{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 在 Rental Listing Inquiries 数据上练习分类方法"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 首先 import 必要的模块\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "#竞赛的评价指标为logloss\n",
    "from sklearn.metrics import log_loss  \n",
    "\n",
    "from matplotlib import pyplot as plt\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 读取数据 & 数据探索"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>building_id</th>\n",
       "      <th>created</th>\n",
       "      <th>description</th>\n",
       "      <th>display_address</th>\n",
       "      <th>features</th>\n",
       "      <th>interest_level</th>\n",
       "      <th>latitude</th>\n",
       "      <th>listing_id</th>\n",
       "      <th>longitude</th>\n",
       "      <th>manager_id</th>\n",
       "      <th>photos</th>\n",
       "      <th>price</th>\n",
       "      <th>street_address</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1.5</td>\n",
       "      <td>3</td>\n",
       "      <td>53a5b119ba8f7b61d4e010512e0dfc85</td>\n",
       "      <td>2016-06-24 07:54:24</td>\n",
       "      <td>A Brand New 3 Bedroom 1.5 bath ApartmentEnjoy ...</td>\n",
       "      <td>Metropolitan Avenue</td>\n",
       "      <td>[]</td>\n",
       "      <td>medium</td>\n",
       "      <td>40.7145</td>\n",
       "      <td>7211212</td>\n",
       "      <td>-73.9425</td>\n",
       "      <td>5ba989232d0489da1b5f2c45f6688adc</td>\n",
       "      <td>[https://photos.renthop.com/2/7211212_1ed4542e...</td>\n",
       "      <td>3000</td>\n",
       "      <td>792 Metropolitan Avenue</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10000</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>c5c8a357cba207596b04d1afd1e4f130</td>\n",
       "      <td>2016-06-12 12:19:27</td>\n",
       "      <td></td>\n",
       "      <td>Columbus Avenue</td>\n",
       "      <td>[Doorman, Elevator, Fitness Center, Cats Allow...</td>\n",
       "      <td>low</td>\n",
       "      <td>40.7947</td>\n",
       "      <td>7150865</td>\n",
       "      <td>-73.9667</td>\n",
       "      <td>7533621a882f71e25173b27e3139d83d</td>\n",
       "      <td>[https://photos.renthop.com/2/7150865_be3306c5...</td>\n",
       "      <td>5465</td>\n",
       "      <td>808 Columbus Avenue</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100004</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>c3ba40552e2120b0acfc3cb5730bb2aa</td>\n",
       "      <td>2016-04-17 03:26:41</td>\n",
       "      <td>Top Top West Village location, beautiful Pre-w...</td>\n",
       "      <td>W 13 Street</td>\n",
       "      <td>[Laundry In Building, Dishwasher, Hardwood Flo...</td>\n",
       "      <td>high</td>\n",
       "      <td>40.7388</td>\n",
       "      <td>6887163</td>\n",
       "      <td>-74.0018</td>\n",
       "      <td>d9039c43983f6e564b1482b273bd7b01</td>\n",
       "      <td>[https://photos.renthop.com/2/6887163_de85c427...</td>\n",
       "      <td>2850</td>\n",
       "      <td>241 W 13 Street</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100007</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>28d9ad350afeaab8027513a3e52ac8d5</td>\n",
       "      <td>2016-04-18 02:22:02</td>\n",
       "      <td>Building Amenities - Garage - Garden - fitness...</td>\n",
       "      <td>East 49th Street</td>\n",
       "      <td>[Hardwood Floors, No Fee]</td>\n",
       "      <td>low</td>\n",
       "      <td>40.7539</td>\n",
       "      <td>6888711</td>\n",
       "      <td>-73.9677</td>\n",
       "      <td>1067e078446a7897d2da493d2f741316</td>\n",
       "      <td>[https://photos.renthop.com/2/6888711_6e660cee...</td>\n",
       "      <td>3275</td>\n",
       "      <td>333 East 49th Street</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100013</th>\n",
       "      <td>1.0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>2016-04-28 01:32:41</td>\n",
       "      <td>Beautifully renovated 3 bedroom flex 4 bedroom...</td>\n",
       "      <td>West 143rd Street</td>\n",
       "      <td>[Pre-War]</td>\n",
       "      <td>low</td>\n",
       "      <td>40.8241</td>\n",
       "      <td>6934781</td>\n",
       "      <td>-73.9493</td>\n",
       "      <td>98e13ad4b495b9613cef886d79a6291f</td>\n",
       "      <td>[https://photos.renthop.com/2/6934781_1fa4b41a...</td>\n",
       "      <td>3350</td>\n",
       "      <td>500 West 143rd Street</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        bathrooms  bedrooms                       building_id  \\\n",
       "10            1.5         3  53a5b119ba8f7b61d4e010512e0dfc85   \n",
       "10000         1.0         2  c5c8a357cba207596b04d1afd1e4f130   \n",
       "100004        1.0         1  c3ba40552e2120b0acfc3cb5730bb2aa   \n",
       "100007        1.0         1  28d9ad350afeaab8027513a3e52ac8d5   \n",
       "100013        1.0         4                                 0   \n",
       "\n",
       "                    created  \\\n",
       "10      2016-06-24 07:54:24   \n",
       "10000   2016-06-12 12:19:27   \n",
       "100004  2016-04-17 03:26:41   \n",
       "100007  2016-04-18 02:22:02   \n",
       "100013  2016-04-28 01:32:41   \n",
       "\n",
       "                                              description  \\\n",
       "10      A Brand New 3 Bedroom 1.5 bath ApartmentEnjoy ...   \n",
       "10000                                                       \n",
       "100004  Top Top West Village location, beautiful Pre-w...   \n",
       "100007  Building Amenities - Garage - Garden - fitness...   \n",
       "100013  Beautifully renovated 3 bedroom flex 4 bedroom...   \n",
       "\n",
       "            display_address  \\\n",
       "10      Metropolitan Avenue   \n",
       "10000       Columbus Avenue   \n",
       "100004          W 13 Street   \n",
       "100007     East 49th Street   \n",
       "100013    West 143rd Street   \n",
       "\n",
       "                                                 features interest_level  \\\n",
       "10                                                     []         medium   \n",
       "10000   [Doorman, Elevator, Fitness Center, Cats Allow...            low   \n",
       "100004  [Laundry In Building, Dishwasher, Hardwood Flo...           high   \n",
       "100007                          [Hardwood Floors, No Fee]            low   \n",
       "100013                                          [Pre-War]            low   \n",
       "\n",
       "        latitude  listing_id  longitude                        manager_id  \\\n",
       "10       40.7145     7211212   -73.9425  5ba989232d0489da1b5f2c45f6688adc   \n",
       "10000    40.7947     7150865   -73.9667  7533621a882f71e25173b27e3139d83d   \n",
       "100004   40.7388     6887163   -74.0018  d9039c43983f6e564b1482b273bd7b01   \n",
       "100007   40.7539     6888711   -73.9677  1067e078446a7897d2da493d2f741316   \n",
       "100013   40.8241     6934781   -73.9493  98e13ad4b495b9613cef886d79a6291f   \n",
       "\n",
       "                                                   photos  price  \\\n",
       "10      [https://photos.renthop.com/2/7211212_1ed4542e...   3000   \n",
       "10000   [https://photos.renthop.com/2/7150865_be3306c5...   5465   \n",
       "100004  [https://photos.renthop.com/2/6887163_de85c427...   2850   \n",
       "100007  [https://photos.renthop.com/2/6888711_6e660cee...   3275   \n",
       "100013  [https://photos.renthop.com/2/6934781_1fa4b41a...   3350   \n",
       "\n",
       "                 street_address  \n",
       "10      792 Metropolitan Avenue  \n",
       "10000       808 Columbus Avenue  \n",
       "100004          241 W 13 Street  \n",
       "100007     333 East 49th Street  \n",
       "100013    500 West 143rd Street  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 读取训练数据\n",
    "# path to where the data lies\n",
    "dpath = '../../data/'\n",
    "train = pd.read_json(dpath +\"RentListingInquries_train.json\")\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 49352 entries, 10 to 99994\n",
      "Data columns (total 15 columns):\n",
      "bathrooms          49352 non-null float64\n",
      "bedrooms           49352 non-null int64\n",
      "building_id        49352 non-null object\n",
      "created            49352 non-null object\n",
      "description        49352 non-null object\n",
      "display_address    49352 non-null object\n",
      "features           49352 non-null object\n",
      "interest_level     49352 non-null object\n",
      "latitude           49352 non-null float64\n",
      "listing_id         49352 non-null int64\n",
      "longitude          49352 non-null float64\n",
      "manager_id         49352 non-null object\n",
      "photos             49352 non-null object\n",
      "price              49352 non-null int64\n",
      "street_address     49352 non-null object\n",
      "dtypes: float64(3), int64(3), object(9)\n",
      "memory usage: 6.0+ MB\n"
     ]
    }
   ],
   "source": [
    "train.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 特征含义\n",
    "bathrooms: 浴室的数量\n",
    "\n",
    "bedrooms: 卧室的数量\n",
    "\n",
    "building_id：\n",
    "\n",
    "created：发布时间\n",
    "\n",
    "description：一些描述\n",
    "\n",
    "display_address：展出地址\n",
    "\n",
    "features: 公寓的一些特征\n",
    "\n",
    "latitude：纬度\n",
    "\n",
    "listing_id\n",
    "\n",
    "longitude：经度\n",
    "\n",
    "manager_id：管理ID\n",
    "\n",
    "photos: 租房图片集\n",
    "\n",
    "price: 美元\n",
    "\n",
    "street_address：街道地址\n",
    "\n",
    "interest_level: 目标变量，受欢迎程度. 有三个类: ‘high’, ‘medium’, ‘low’"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "bathrooms          0\n",
       "bedrooms           0\n",
       "building_id        0\n",
       "created            0\n",
       "description        0\n",
       "display_address    0\n",
       "features           0\n",
       "interest_level     0\n",
       "latitude           0\n",
       "listing_id         0\n",
       "longitude          0\n",
       "manager_id         0\n",
       "photos             0\n",
       "price              0\n",
       "street_address     0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bathrooms = 313\n",
      "bedrooms = 9475\n",
      "building_id = 0\n",
      "created = 0\n",
      "description = 0\n",
      "display_address = 0\n",
      "features = 0\n",
      "interest_level = 0\n",
      "latitude = 12\n",
      "listing_id = 0\n",
      "longitude = 12\n",
      "manager_id = 0\n",
      "photos = 0\n",
      "price = 0\n",
      "street_address = 0\n"
     ]
    }
   ],
   "source": [
    "for a in train.columns:\n",
    "    print('%s = %d'% (a ,len(train[train[a]==0])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>latitude</th>\n",
       "      <th>listing_id</th>\n",
       "      <th>longitude</th>\n",
       "      <th>price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>49352.00000</td>\n",
       "      <td>49352.000000</td>\n",
       "      <td>49352.000000</td>\n",
       "      <td>4.935200e+04</td>\n",
       "      <td>49352.000000</td>\n",
       "      <td>4.935200e+04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>1.21218</td>\n",
       "      <td>1.541640</td>\n",
       "      <td>40.741545</td>\n",
       "      <td>7.024055e+06</td>\n",
       "      <td>-73.955716</td>\n",
       "      <td>3.830174e+03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.50142</td>\n",
       "      <td>1.115018</td>\n",
       "      <td>0.638535</td>\n",
       "      <td>1.262746e+05</td>\n",
       "      <td>1.177912</td>\n",
       "      <td>2.206687e+04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>6.811957e+06</td>\n",
       "      <td>-118.271000</td>\n",
       "      <td>4.300000e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.00000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>40.728300</td>\n",
       "      <td>6.915888e+06</td>\n",
       "      <td>-73.991700</td>\n",
       "      <td>2.500000e+03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>1.00000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>40.751800</td>\n",
       "      <td>7.021070e+06</td>\n",
       "      <td>-73.977900</td>\n",
       "      <td>3.150000e+03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.00000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>40.774300</td>\n",
       "      <td>7.128733e+06</td>\n",
       "      <td>-73.954800</td>\n",
       "      <td>4.100000e+03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>10.00000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>44.883500</td>\n",
       "      <td>7.753784e+06</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4.490000e+06</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         bathrooms      bedrooms      latitude    listing_id     longitude  \\\n",
       "count  49352.00000  49352.000000  49352.000000  4.935200e+04  49352.000000   \n",
       "mean       1.21218      1.541640     40.741545  7.024055e+06    -73.955716   \n",
       "std        0.50142      1.115018      0.638535  1.262746e+05      1.177912   \n",
       "min        0.00000      0.000000      0.000000  6.811957e+06   -118.271000   \n",
       "25%        1.00000      1.000000     40.728300  6.915888e+06    -73.991700   \n",
       "50%        1.00000      1.000000     40.751800  7.021070e+06    -73.977900   \n",
       "75%        1.00000      2.000000     40.774300  7.128733e+06    -73.954800   \n",
       "max       10.00000      8.000000     44.883500  7.753784e+06      0.000000   \n",
       "\n",
       "              price  \n",
       "count  4.935200e+04  \n",
       "mean   3.830174e+03  \n",
       "std    2.206687e+04  \n",
       "min    4.300000e+01  \n",
       "25%    2.500000e+03  \n",
       "50%    3.150000e+03  \n",
       "75%    4.100000e+03  \n",
       "max    4.490000e+06  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## 各属性的统计特性\n",
    "train.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>building_id</th>\n",
       "      <th>created</th>\n",
       "      <th>description</th>\n",
       "      <th>display_address</th>\n",
       "      <th>features</th>\n",
       "      <th>latitude</th>\n",
       "      <th>listing_id</th>\n",
       "      <th>longitude</th>\n",
       "      <th>manager_id</th>\n",
       "      <th>photos</th>\n",
       "      <th>price</th>\n",
       "      <th>street_address</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>79780be1514f645d7e6be99a3de696c5</td>\n",
       "      <td>2016-06-11 05:29:41</td>\n",
       "      <td>Large with awesome terrace--accessible via bed...</td>\n",
       "      <td>Suffolk Street</td>\n",
       "      <td>[Elevator, Laundry in Building, Laundry in Uni...</td>\n",
       "      <td>40.7185</td>\n",
       "      <td>7142618</td>\n",
       "      <td>-73.9865</td>\n",
       "      <td>b1b1852c416d78d7765d746cb1b8921f</td>\n",
       "      <td>[https://photos.renthop.com/2/7142618_1c45a2c8...</td>\n",
       "      <td>2950</td>\n",
       "      <td>99 Suffolk Street</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2016-06-24 06:36:34</td>\n",
       "      <td>Prime Soho - between Bleecker and Houston - Ne...</td>\n",
       "      <td>Thompson Street</td>\n",
       "      <td>[Pre-War, Dogs Allowed, Cats Allowed]</td>\n",
       "      <td>40.7278</td>\n",
       "      <td>7210040</td>\n",
       "      <td>-74.0000</td>\n",
       "      <td>d0b5648017832b2427eeb9956d966a14</td>\n",
       "      <td>[https://photos.renthop.com/2/7210040_d824cc71...</td>\n",
       "      <td>2850</td>\n",
       "      <td>176 Thompson Street</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>3dbbb69fd52e0d25131aa1cd459c87eb</td>\n",
       "      <td>2016-06-03 04:29:40</td>\n",
       "      <td>New York chic has reached a new level ...</td>\n",
       "      <td>101 East 10th Street</td>\n",
       "      <td>[Doorman, Elevator, No Fee]</td>\n",
       "      <td>40.7306</td>\n",
       "      <td>7103890</td>\n",
       "      <td>-73.9890</td>\n",
       "      <td>9ca6f3baa475c37a3b3521a394d65467</td>\n",
       "      <td>[https://photos.renthop.com/2/7103890_85b33077...</td>\n",
       "      <td>3758</td>\n",
       "      <td>101 East 10th Street</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1000</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>783d21d013a7e655bddc4ed0d461cc5e</td>\n",
       "      <td>2016-06-11 06:17:35</td>\n",
       "      <td>Step into this fantastic new Construction in t...</td>\n",
       "      <td>South Third Street\\r</td>\n",
       "      <td>[Roof Deck, Balcony, Elevator, Laundry in Buil...</td>\n",
       "      <td>40.7109</td>\n",
       "      <td>7143442</td>\n",
       "      <td>-73.9571</td>\n",
       "      <td>0b9d5db96db8472d7aeb67c67338c4d2</td>\n",
       "      <td>[https://photos.renthop.com/2/7143442_0879e9e0...</td>\n",
       "      <td>3300</td>\n",
       "      <td>251  South Third Street\\r</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100000</th>\n",
       "      <td>2.0</td>\n",
       "      <td>2</td>\n",
       "      <td>6134e7c4dd1a98d9aee36623c9872b49</td>\n",
       "      <td>2016-04-12 05:24:17</td>\n",
       "      <td>~Take a stroll in Central Park, enjoy the ente...</td>\n",
       "      <td>Midtown West, 8th Ave</td>\n",
       "      <td>[Common Outdoor Space, Cats Allowed, Dogs Allo...</td>\n",
       "      <td>40.7650</td>\n",
       "      <td>6860601</td>\n",
       "      <td>-73.9845</td>\n",
       "      <td>b5eda0eb31b042ce2124fd9e9fcfce2f</td>\n",
       "      <td>[https://photos.renthop.com/2/6860601_c96164d8...</td>\n",
       "      <td>4900</td>\n",
       "      <td>260 West 54th Street</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        bathrooms  bedrooms                       building_id  \\\n",
       "0             1.0         1  79780be1514f645d7e6be99a3de696c5   \n",
       "1             1.0         2                                 0   \n",
       "100           1.0         1  3dbbb69fd52e0d25131aa1cd459c87eb   \n",
       "1000          1.0         2  783d21d013a7e655bddc4ed0d461cc5e   \n",
       "100000        2.0         2  6134e7c4dd1a98d9aee36623c9872b49   \n",
       "\n",
       "                    created  \\\n",
       "0       2016-06-11 05:29:41   \n",
       "1       2016-06-24 06:36:34   \n",
       "100     2016-06-03 04:29:40   \n",
       "1000    2016-06-11 06:17:35   \n",
       "100000  2016-04-12 05:24:17   \n",
       "\n",
       "                                              description  \\\n",
       "0       Large with awesome terrace--accessible via bed...   \n",
       "1       Prime Soho - between Bleecker and Houston - Ne...   \n",
       "100             New York chic has reached a new level ...   \n",
       "1000    Step into this fantastic new Construction in t...   \n",
       "100000  ~Take a stroll in Central Park, enjoy the ente...   \n",
       "\n",
       "              display_address  \\\n",
       "0              Suffolk Street   \n",
       "1             Thompson Street   \n",
       "100      101 East 10th Street   \n",
       "1000     South Third Street\\r   \n",
       "100000  Midtown West, 8th Ave   \n",
       "\n",
       "                                                 features  latitude  \\\n",
       "0       [Elevator, Laundry in Building, Laundry in Uni...   40.7185   \n",
       "1                   [Pre-War, Dogs Allowed, Cats Allowed]   40.7278   \n",
       "100                           [Doorman, Elevator, No Fee]   40.7306   \n",
       "1000    [Roof Deck, Balcony, Elevator, Laundry in Buil...   40.7109   \n",
       "100000  [Common Outdoor Space, Cats Allowed, Dogs Allo...   40.7650   \n",
       "\n",
       "        listing_id  longitude                        manager_id  \\\n",
       "0          7142618   -73.9865  b1b1852c416d78d7765d746cb1b8921f   \n",
       "1          7210040   -74.0000  d0b5648017832b2427eeb9956d966a14   \n",
       "100        7103890   -73.9890  9ca6f3baa475c37a3b3521a394d65467   \n",
       "1000       7143442   -73.9571  0b9d5db96db8472d7aeb67c67338c4d2   \n",
       "100000     6860601   -73.9845  b5eda0eb31b042ce2124fd9e9fcfce2f   \n",
       "\n",
       "                                                   photos  price  \\\n",
       "0       [https://photos.renthop.com/2/7142618_1c45a2c8...   2950   \n",
       "1       [https://photos.renthop.com/2/7210040_d824cc71...   2850   \n",
       "100     [https://photos.renthop.com/2/7103890_85b33077...   3758   \n",
       "1000    [https://photos.renthop.com/2/7143442_0879e9e0...   3300   \n",
       "100000  [https://photos.renthop.com/2/6860601_c96164d8...   4900   \n",
       "\n",
       "                   street_address  \n",
       "0               99 Suffolk Street  \n",
       "1             176 Thompson Street  \n",
       "100          101 East 10th Street  \n",
       "1000    251  South Third Street\\r  \n",
       "100000       260 West 54th Street  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 读取测试数据\n",
    "# path to where the data lies\n",
    "dpath = '../../data/'\n",
    "test = pd.read_json(dpath +\"RentListingInquries_test.json\")\n",
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 74659 entries, 0 to 99999\n",
      "Data columns (total 14 columns):\n",
      "bathrooms          74659 non-null float64\n",
      "bedrooms           74659 non-null int64\n",
      "building_id        74659 non-null object\n",
      "created            74659 non-null object\n",
      "description        74659 non-null object\n",
      "display_address    74659 non-null object\n",
      "features           74659 non-null object\n",
      "latitude           74659 non-null float64\n",
      "listing_id         74659 non-null int64\n",
      "longitude          74659 non-null float64\n",
      "manager_id         74659 non-null object\n",
      "photos             74659 non-null object\n",
      "price              74659 non-null int64\n",
      "street_address     74659 non-null object\n",
      "dtypes: float64(3), int64(3), object(8)\n",
      "memory usage: 8.5+ MB\n"
     ]
    }
   ],
   "source": [
    "test.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 计算资源原因，只取500个样本进行试验"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "train=train[0:500]\n",
    "test=test[0:500]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy import sparse\n",
    "from sklearn import preprocessing\n",
    "from sklearn.feature_extraction.text import CountVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "features_to_use = [\"bathrooms\",\"bedrooms\",\"latitude\",\"longitude\",\"price\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 照片数量(num_photos)\n",
    "train['num_photos']=train['photos'].apply(len)\n",
    "test['num_photos']=test['photos'].apply(len)\n",
    "# 特征数量\n",
    "train['num_features']=train['features'].apply(len)\n",
    "test['num_features']=test['features'].apply(len)\n",
    "# 描述词汇数量\n",
    "train['num_description_words'] = train['description'].apply(lambda x: len(x.split(\" \")))\n",
    "test['num_description_words'] = test['description'].apply(lambda x: len(x.split(\" \")))\n",
    "#把创建的时间分解为多个特征        \n",
    "train['created']=pd.to_datetime(train['created'])\n",
    "test['created']=pd.to_datetime(test['created'])\n",
    " \n",
    "#让我们从时间中分解出一些特征，比如年，月，日，时\n",
    "#年\n",
    "train['created_year'] = train['created'].dt.year\n",
    "test['created_year'] = test['created'].dt.year\n",
    "#月\n",
    "train['created_month'] = train['created'].dt.month\n",
    "test['created_month'] = test['created'].dt.month\n",
    "#日\n",
    "train['created_day'] = train['created'].dt.day\n",
    "test['created_day'] = test['created'].dt.day\n",
    "#时\n",
    "train['created_hour'] = train['created'].dt.hour\n",
    "test['created_hour'] = test['created'].dt.hour\n",
    "#把这些特征都放到所需特征列表中（上面已经创建，并加入了数值型特征） \n",
    "features_to_use.extend([\"num_photos\",\"num_features\",\"num_description_words\",\"created_year\",\"created_month\",\"created_day\",\"created_hour\",\"listing_id\"])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 对字符串这类变量先做类别编码，后续考虑做独热编码"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "categorical = [\"display_address\",\"manager_id\",'building_id',\"street_address\"]\n",
    "for f in categorical:\n",
    "    if train[f].dtype == 'object':\n",
    "        lbl = preprocessing.LabelEncoder()\n",
    "        lbl.fit(list(train[f].values)+list(test[f].values))\n",
    "        train[f] = lbl.transform(list(train[f].values))\n",
    "        test[f] = lbl.transform(list(test[f].values))\n",
    "        features_to_use.append(f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 将对房源的描述变为词库，进一步转化为词向量"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10                                                        \n",
      "10000    Doorman Elevator Fitness_Center Cats_Allowed D...\n",
      "Name: features, dtype: object\n",
      "0    Elevator Laundry_in_Building Laundry_in_Unit D...\n",
      "1                    Pre-War Dogs_Allowed Cats_Allowed\n",
      "Name: features, dtype: object\n"
     ]
    }
   ],
   "source": [
    "\n",
    "train[\"features\"] = train[\"features\"].apply(lambda x:\" \".join([\"_\".join(i.split(\" \"))for i in x]))\n",
    "print (train['features'].head(2))\n",
    "test['features'] = test[\"features\"].apply(lambda x: \" \".join([\"_\".join(i.split(\" \"))for i in x]))\n",
    "print (test['features'].head(2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "tfidf = CountVectorizer(stop_words =\"english\",max_features=200)\n",
    "tr_sparse = tfidf.fit_transform(train[\"features\"])\n",
    "te_sparse = tfidf.transform(test[\"features\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = sparse.hstack([train[features_to_use],tr_sparse]).tocsr()\n",
    "test = sparse.hstack([test[features_to_use],te_sparse]).tocsr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(500, 140) (500, 140) (500,)\n"
     ]
    }
   ],
   "source": [
    "target_num_map = {'high':0 , 'medium':1 , 'low':2}\n",
    "y_train = np.array(train['interest_level'].apply(lambda x: target_num_map[x]))\n",
    "print (X_train.shape,test.shape,y_train.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 数据集划分"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def data_set_split(X,Y):\n",
    "    from sklearn.cross_validation import train_test_split\n",
    "    #random_state=42 保证每次分组一致，保证模型稳定\n",
    "    X_train, X_test, y_train,y_test = train_test_split(X, Y, test_size=0.2) \n",
    "    return X_train, X_test, y_train, y_test\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "X_train, X_test, y_train, y_test=data_set_split(X_train,y_train)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 数据预处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 数据标准化\n",
    "#from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "# 初始化特征的标准化器\n",
    "#ss_X = StandardScaler()\n",
    "\n",
    "# 分别对训练和测试数据的特征进行标准化处理\n",
    "#X_train = ss_X.fit_transform(X_train)\n",
    "#X_test = ss_X.transform(X_test)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "将0.2的数据作为测试集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(400, 140) (100, 140)\n"
     ]
    }
   ],
   "source": [
    "print(X_train.shape,X_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 模型训练"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### default Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "lr= LogisticRegression()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# 交叉验证用于评估模型性能和进行参数调优（模型选择）\n",
    "#分类任务中交叉验证缺省是采用StratifiedKFold\n",
    "from sklearn.cross_validation import cross_val_score\n",
    "loss = cross_val_score(lr, X_train, y_train, cv=5, scoring='neg_log_loss')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "logloss of each fold is:  [0.80527974 0.7671725  0.80183464 0.77380223 0.77206277]\n",
      "cv logloss is: 0.7840303752329437\n"
     ]
    }
   ],
   "source": [
    "print ('logloss of each fold is: ',-loss)\n",
    "print ('cv logloss is:', -loss.mean())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 正则化的 Logistic Regression及参数调优"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "logistic回归的需要调整超参数有：C（正则系数，一般在log域（取log后的值）均匀设置候选参数）和正则函数penalty（L2/L1） \n",
    "目标函数为：J = sum(logloss(f(xi), yi)) + C* penalty \n",
    "\n",
    "在sklearn框架下，不同学习器的参数调整步骤相同：\n",
    "设置候选参数集合\n",
    "调用GridSearchCV\n",
    "调用fit"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 用LogisticRegressionCV实现正则化的 Logistic Regression"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### L1正则"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegressionCV(Cs=[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7, 1.8, 1.9, 2.0, 2.1, 2.2, 2.3, 2.4, 2.5, 2.6, 2.7, 2.8, 2.9, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6, 3.7, 3.8, 3.9, 4.0, 4.1, 4.2, 4.3, 4.4, 4.5, 4.6, 4.7, 4.8, 4.9, 5.0, 5.1, 5.2, 5.3, 5.4, 5.5, 5.6, 5.7, 5.8, 5.9, 6.0, 6.1, 6.2, 6.3, 6.4, 6.5, 6.6, 6.7, 6.8, 6.9, 7.0, 7.1, 7.2, 7.3, 7.4, 7.5, 7.6, 7.7, 7.8, 7.9, 8.0, 8.1, 8.2, 8.3, 8.4, 8.5, 8.6, 8.7, 8.8, 8.9, 9.0, 9.1, 9.2, 9.3, 9.4, 9.5, 9.6, 9.7, 9.8, 9.9],\n",
       "           class_weight=None, cv=5, dual=False, fit_intercept=True,\n",
       "           intercept_scaling=1.0, max_iter=100, multi_class='ovr',\n",
       "           n_jobs=1, penalty='l1', random_state=None, refit=True,\n",
       "           scoring='neg_log_loss', solver='liblinear', tol=0.0001,\n",
       "           verbose=0)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegressionCV\n",
    "\n",
    "#Cs = [1e-06, 1e-05, 0.0001, 0.001, 0.01, 0.1,1,10,100,1000]\n",
    "Cs=[i/10 for i in range(1,100)]\n",
    "# LogisticRegressionCV比GridSearchCV快\n",
    "lrcv_L1 = LogisticRegressionCV(Cs=Cs, cv = 5, scoring='neg_log_loss', penalty='l1', solver='liblinear')\n",
    "lrcv_L1.fit(X_train, y_train)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: array([[-0.28315036, -0.27677367, -0.28224789, -0.27943279, -0.27476398,\n",
       "         -0.27362478, -0.27350612, -0.27473198, -0.274424  , -0.27466628,\n",
       "         -0.28224883, -0.27722101, -0.27923827, -0.28113695, -0.2839096 ,\n",
       "         -0.28627463, -0.288545  , -0.29107023, -0.29437894, -0.29892599,\n",
       "         -0.29814703, -0.30190568, -0.30305868, -0.3050048 , -0.30719139,\n",
       "         -0.30915226, -0.31113706, -0.31320841, -0.31557381, -0.31706089,\n",
       "         -0.31874583, -0.32109725, -0.325404  , -0.32541969, -0.32846674,\n",
       "         -0.32980726, -0.33263749, -0.33564118, -0.33651302, -0.33926974,\n",
       "         -0.34004291, -0.3423762 , -0.34685097, -0.3467379 , -0.34839428,\n",
       "         -0.35054321, -0.35169222, -0.35372974, -0.35704471, -0.35559512,\n",
       "         -0.35960838, -0.36074846, -0.36005359, -0.36144918, -0.36310097,\n",
       "         -0.36748456, -0.36883973, -0.36998209, -0.37102039, -0.3732604 ,\n",
       "         -0.37145319, -0.37628108, -0.37779057, -0.37576302, -0.37802428,\n",
       "         -0.38062148, -0.3829548 , -0.3810021 , -0.38590266, -0.38754155,\n",
       "         -0.38804605, -0.38945847, -0.38644622, -0.39247835, -0.39284655,\n",
       "         -0.39411467, -0.39324282, -0.39604119, -0.39474635, -0.39495302,\n",
       "         -0.39421001, -0.40072096, -0.40046623, -0.39703468, -0.40439839,\n",
       "         -0.40536117, -0.4055359 , -0.40158867, -0.40656635, -0.40969652,\n",
       "         -0.4031565 , -0.41109019, -0.41163626, -0.40658454, -0.40801057,\n",
       "         -0.4120122 , -0.41004558, -0.41648391, -0.41111911],\n",
       "        [-0.25481018, -0.24994667, -0.24462168, -0.24532092, -0.2488165 ,\n",
       "         -0.2530207 , -0.25675449, -0.26008722, -0.26315777, -0.26551039,\n",
       "         -0.26619715, -0.27015163, -0.26999936, -0.26950939, -0.27134022,\n",
       "         -0.27645309, -0.27608015, -0.28105366, -0.2829492 , -0.28289444,\n",
       "         -0.28590486, -0.28887864, -0.29017077, -0.29144668, -0.28876074,\n",
       "         -0.29481572, -0.29471889, -0.29561265, -0.29714764, -0.29785283,\n",
       "         -0.29870925, -0.29557164, -0.30006912, -0.30027767, -0.29211863,\n",
       "         -0.30229859, -0.30249502, -0.30336884, -0.29360502, -0.30420815,\n",
       "         -0.30514773, -0.2981852 , -0.29656016, -0.30612513, -0.30681656,\n",
       "         -0.30212845, -0.29929238, -0.3079245 , -0.30813893, -0.30869396,\n",
       "         -0.30901226, -0.30937763, -0.30987547, -0.31029346, -0.31072568,\n",
       "         -0.302908  , -0.31181476, -0.31241803, -0.31323417, -0.3138871 ,\n",
       "         -0.31465844, -0.31539054, -0.31607376, -0.30624464, -0.3079476 ,\n",
       "         -0.30772132, -0.31871003, -0.31087167, -0.32025628, -0.3093947 ,\n",
       "         -0.31506513, -0.3221978 , -0.31065167, -0.3233942 , -0.32404661,\n",
       "         -0.3155403 , -0.32522556, -0.31606168, -0.32641871, -0.32693479,\n",
       "         -0.32752487, -0.32816275, -0.32873369, -0.32006057, -0.31797288,\n",
       "         -0.33030702, -0.33016229, -0.33136628, -0.33174698, -0.33229086,\n",
       "         -0.3327839 , -0.3333557 , -0.33373717, -0.3342314 , -0.33474324,\n",
       "         -0.32445696, -0.32619115, -0.33617238, -0.3365869 ],\n",
       "        [-0.34017798, -0.33521977, -0.33490695, -0.33245509, -0.33186197,\n",
       "         -0.33437742, -0.33714631, -0.3382799 , -0.33668396, -0.33865295,\n",
       "         -0.34310094, -0.3390631 , -0.34469039, -0.34159727, -0.34695077,\n",
       "         -0.34736025, -0.34842554, -0.35190002, -0.3541051 , -0.35348699,\n",
       "         -0.35440313, -0.35847739, -0.35224755, -0.35148668, -0.3647181 ,\n",
       "         -0.36487768, -0.3670071 , -0.37218324, -0.35780153, -0.37603149,\n",
       "         -0.37455689, -0.3758396 , -0.37854637, -0.37914096, -0.3845513 ,\n",
       "         -0.38461957, -0.38373379, -0.38473351, -0.38850811, -0.38624835,\n",
       "         -0.34140581, -0.3906929 , -0.3889556 , -0.39725972, -0.39175347,\n",
       "         -0.40042062, -0.36221309, -0.38312334, -0.39662288, -0.40032865,\n",
       "         -0.40493099, -0.3768033 , -0.40144507, -0.40311347, -0.35374371,\n",
       "         -0.41590473, -0.41274308, -0.41160705, -0.41963553, -0.39999703,\n",
       "         -0.41737334, -0.41426737, -0.42107621, -0.41797393, -0.40895139,\n",
       "         -0.38566832, -0.41171084, -0.41347557, -0.43068137, -0.42820254,\n",
       "         -0.42557687, -0.43154403, -0.43061809, -0.4344018 , -0.42722799,\n",
       "         -0.43546064, -0.43023913, -0.40776814, -0.43860035, -0.43919773,\n",
       "         -0.42148481, -0.37969604, -0.42555502, -0.44015659, -0.42242339,\n",
       "         -0.44213816, -0.43122288, -0.44078084, -0.43618178, -0.43774308,\n",
       "         -0.44608998, -0.44782791, -0.40841729, -0.43414989, -0.44431989,\n",
       "         -0.44498768, -0.45092171, -0.43831709, -0.44624956],\n",
       "        [-0.24576902, -0.24379236, -0.24861833, -0.25221955, -0.25214908,\n",
       "         -0.25056735, -0.25101839, -0.25395973, -0.25313273, -0.25261395,\n",
       "         -0.25453487, -0.25718499, -0.25803065, -0.25868559, -0.25689506,\n",
       "         -0.25772996, -0.26084798, -0.2627925 , -0.26116697, -0.25976119,\n",
       "         -0.26561373, -0.2674126 , -0.26652507, -0.26718738, -0.27069766,\n",
       "         -0.26735663, -0.2696334 , -0.27389902, -0.27432216, -0.27422384,\n",
       "         -0.27523451, -0.27513629, -0.27488396, -0.27627231, -0.277276  ,\n",
       "         -0.27568674, -0.27577361, -0.27824371, -0.27319629, -0.27945746,\n",
       "         -0.27852699, -0.28058829, -0.2813593 , -0.25335054, -0.28189879,\n",
       "         -0.28289092, -0.28322414, -0.28357506, -0.2842789 , -0.2842362 ,\n",
       "         -0.28527998, -0.28555546, -0.28549641, -0.28511183, -0.28621624,\n",
       "         -0.28777136, -0.28633811, -0.28824143, -0.28891275, -0.28863928,\n",
       "         -0.28988445, -0.26958504, -0.2901794 , -0.29054494, -0.2904246 ,\n",
       "         -0.29050485, -0.28824098, -0.29257465, -0.29273104, -0.29292057,\n",
       "         -0.29344333, -0.28607915, -0.29405369, -0.29235047, -0.29551664,\n",
       "         -0.29622254, -0.29294811, -0.29177196, -0.29415155, -0.29604616,\n",
       "         -0.29373769, -0.2979251 , -0.29847077, -0.26897361, -0.29490269,\n",
       "         -0.29922805, -0.29613498, -0.26369853, -0.2937418 , -0.29806892,\n",
       "         -0.30054393, -0.2972431 , -0.29591986, -0.2973175 , -0.30076253,\n",
       "         -0.30235146, -0.29811538, -0.30261519, -0.30284385],\n",
       "        [-0.2798611 , -0.27341841, -0.26767281, -0.26892227, -0.259979  ,\n",
       "         -0.25714066, -0.2552753 , -0.25522745, -0.25787834, -0.25854032,\n",
       "         -0.2557522 , -0.2588949 , -0.25547326, -0.25832711, -0.25446945,\n",
       "         -0.25292701, -0.25561295, -0.2540503 , -0.25436794, -0.24960894,\n",
       "         -0.25042305, -0.24954165, -0.2526872 , -0.24961056, -0.24946046,\n",
       "         -0.24936946, -0.25274135, -0.24879148, -0.25112675, -0.25287252,\n",
       "         -0.24722913, -0.25229257, -0.25267085, -0.2509407 , -0.24846977,\n",
       "         -0.25218524, -0.24876503, -0.24867258, -0.24870742, -0.2509007 ,\n",
       "         -0.25082899, -0.24829477, -0.25085091, -0.24986733, -0.24892905,\n",
       "         -0.2486719 , -0.24882692, -0.24914133, -0.25167038, -0.25042299,\n",
       "         -0.24871505, -0.25224862, -0.24852605, -0.24870498, -0.25080638,\n",
       "         -0.25004204, -0.24901071, -0.24762335, -0.25303405, -0.25314404,\n",
       "         -0.25024014, -0.2518866 , -0.25351392, -0.24835814, -0.2502275 ,\n",
       "         -0.25370151, -0.25320002, -0.25047476, -0.25390514, -0.25403224,\n",
       "         -0.25410361, -0.25108527, -0.25248237, -0.25436026, -0.2544262 ,\n",
       "         -0.25277912, -0.25445051, -0.25134959, -0.25103929, -0.25023775,\n",
       "         -0.25156038, -0.25339121, -0.25296341, -0.25129618, -0.25020331,\n",
       "         -0.25365219, -0.25527327, -0.25536522, -0.24989519, -0.25416908,\n",
       "         -0.25002199, -0.25568243, -0.25231078, -0.2522064 , -0.25212322,\n",
       "         -0.25045053, -0.25477154, -0.25558699, -0.2555456 ]]),\n",
       " 1: array([[-0.56869168, -0.56332562, -0.55954445, -0.56179402, -0.551609  ,\n",
       "         -0.54417388, -0.54002274, -0.53661405, -0.53189742, -0.53241151,\n",
       "         -0.52451589, -0.53197765, -0.52735798, -0.53047514, -0.526453  ,\n",
       "         -0.52409072, -0.52625563, -0.52130838, -0.52392931, -0.52368177,\n",
       "         -0.52178895, -0.51245472, -0.5253627 , -0.51662139, -0.5133003 ,\n",
       "         -0.52221214, -0.51253776, -0.51479784, -0.52057029, -0.52032578,\n",
       "         -0.522239  , -0.51572218, -0.5253216 , -0.52558787, -0.5089367 ,\n",
       "         -0.4967176 , -0.52436977, -0.5160194 , -0.50932592, -0.51299528,\n",
       "         -0.51621513, -0.52768016, -0.48924164, -0.53314038, -0.52058233,\n",
       "         -0.52900642, -0.52148464, -0.50458663, -0.5328178 , -0.52564908,\n",
       "         -0.5227798 , -0.49519693, -0.52283206, -0.5296947 , -0.5228111 ,\n",
       "         -0.52499134, -0.5233679 , -0.52180646, -0.53382496, -0.51746787,\n",
       "         -0.52809919, -0.53546442, -0.52289231, -0.50868068, -0.53457686,\n",
       "         -0.5266706 , -0.52432535, -0.54600019, -0.51389583, -0.53928753,\n",
       "         -0.53871593, -0.52494634, -0.50825971, -0.52638551, -0.5316579 ,\n",
       "         -0.52765807, -0.54067357, -0.50493799, -0.52813197, -0.52470466,\n",
       "         -0.55324068, -0.53209432, -0.53723254, -0.50602137, -0.54286351,\n",
       "         -0.52988038, -0.53264792, -0.55210997, -0.54720823, -0.50975604,\n",
       "         -0.51879649, -0.55108583, -0.5533322 , -0.5268826 , -0.51437777,\n",
       "         -0.55000146, -0.51319504, -0.51400692, -0.51357919],\n",
       "        [-0.57230178, -0.57449733, -0.579524  , -0.58092574, -0.57947524,\n",
       "         -0.58200051, -0.585532  , -0.59010332, -0.59505269, -0.59940385,\n",
       "         -0.60366855, -0.60908192, -0.61474612, -0.62032305, -0.62580958,\n",
       "         -0.61751166, -0.63846383, -0.64585944, -0.6528461 , -0.65849169,\n",
       "         -0.66385622, -0.66772205, -0.67350839, -0.67731205, -0.68228567,\n",
       "         -0.68599033, -0.68938499, -0.69157415, -0.6958761 , -0.69907267,\n",
       "         -0.70090267, -0.70368648, -0.70761612, -0.70863335, -0.71201211,\n",
       "         -0.7125547 , -0.71454906, -0.6835332 , -0.71669616, -0.71934199,\n",
       "         -0.7212338 , -0.72306115, -0.72441417, -0.7257098 , -0.71943281,\n",
       "         -0.70835414, -0.71693155, -0.70621694, -0.7242059 , -0.73381299,\n",
       "         -0.73295473, -0.72586798, -0.73634968, -0.7296146 , -0.72904463,\n",
       "         -0.73821115, -0.73941484, -0.74209283, -0.74283045, -0.74129223,\n",
       "         -0.74219382, -0.74398497, -0.74310862, -0.74576461, -0.74525058,\n",
       "         -0.74683609, -0.74771821, -0.74760712, -0.74717939, -0.7487777 ,\n",
       "         -0.7495348 , -0.75025044, -0.75129217, -0.74836702, -0.75229554,\n",
       "         -0.7523635 , -0.7498688 , -0.75534913, -0.75391574, -0.75453974,\n",
       "         -0.7458419 , -0.75139074, -0.75825442, -0.75889384, -0.75908827,\n",
       "         -0.75747255, -0.75754438, -0.69961466, -0.75685007, -0.76016132,\n",
       "         -0.75933529, -0.76050519, -0.75585586, -0.75760743, -0.76151184,\n",
       "         -0.7618032 , -0.76207571, -0.7622966 , -0.76301351],\n",
       "        [-0.49824406, -0.50129684, -0.50457011, -0.50513425, -0.50496868,\n",
       "         -0.50573413, -0.50755979, -0.50879464, -0.5137196 , -0.5150366 ,\n",
       "         -0.5143868 , -0.51755676, -0.52385704, -0.52865532, -0.52614773,\n",
       "         -0.53098662, -0.53508195, -0.53427668, -0.53248987, -0.54029571,\n",
       "         -0.53983284, -0.54467464, -0.54492229, -0.54233571, -0.55005165,\n",
       "         -0.54783132, -0.55069954, -0.55641197, -0.55826446, -0.5568225 ,\n",
       "         -0.56291273, -0.567474  , -0.5643183 , -0.56787736, -0.5652885 ,\n",
       "         -0.57138018, -0.56399176, -0.56900677, -0.57214123, -0.57245231,\n",
       "         -0.57153945, -0.57644766, -0.57099268, -0.57497197, -0.57834563,\n",
       "         -0.58307684, -0.59059354, -0.58866784, -0.58218329, -0.5829413 ,\n",
       "         -0.58828321, -0.58897921, -0.59033499, -0.59193264, -0.5915419 ,\n",
       "         -0.59077937, -0.59479947, -0.6037797 , -0.596444  , -0.59364169,\n",
       "         -0.60321734, -0.59622655, -0.60037679, -0.59276618, -0.59767861,\n",
       "         -0.59673132, -0.59678551, -0.59530695, -0.59875519, -0.6150656 ,\n",
       "         -0.60627172, -0.60067939, -0.60518239, -0.60065566, -0.60079337,\n",
       "         -0.60505666, -0.61802891, -0.60341404, -0.61041574, -0.60912282,\n",
       "         -0.61212035, -0.61474366, -0.61253593, -0.60769575, -0.60186696,\n",
       "         -0.60613843, -0.61040485, -0.62349544, -0.61311828, -0.62846829,\n",
       "         -0.61112341, -0.61377991, -0.62030888, -0.6228917 , -0.61418276,\n",
       "         -0.62942548, -0.61358383, -0.61209894, -0.62335617],\n",
       "        [-0.51390046, -0.51414584, -0.51627193, -0.51770292, -0.52680038,\n",
       "         -0.53626471, -0.54665271, -0.5554369 , -0.56511466, -0.57294485,\n",
       "         -0.58305318, -0.59087497, -0.59709391, -0.60496202, -0.61089658,\n",
       "         -0.61718497, -0.61898031, -0.62635098, -0.63024916, -0.63382261,\n",
       "         -0.63761534, -0.64101344, -0.64268178, -0.64735529, -0.64893099,\n",
       "         -0.6519914 , -0.65682167, -0.65821264, -0.6623901 , -0.66539356,\n",
       "         -0.66785443, -0.67072186, -0.6728932 , -0.67753997, -0.67805652,\n",
       "         -0.68064312, -0.68203689, -0.68429865, -0.68471665, -0.68952709,\n",
       "         -0.69167858, -0.69377716, -0.69419735, -0.6958511 , -0.69693801,\n",
       "         -0.69959013, -0.70215944, -0.70501264, -0.70649126, -0.70641415,\n",
       "         -0.70797026, -0.71083803, -0.71367668, -0.71483332, -0.71711865,\n",
       "         -0.71890106, -0.72109592, -0.72214112, -0.72388952, -0.72147379,\n",
       "         -0.72602942, -0.72832827, -0.72972172, -0.72998976, -0.73271731,\n",
       "         -0.73248966, -0.7335577 , -0.73686385, -0.73817184, -0.73872939,\n",
       "         -0.74068588, -0.74228158, -0.74134587, -0.74273444, -0.7454737 ,\n",
       "         -0.7441943 , -0.74779227, -0.7482234 , -0.75012174, -0.75138167,\n",
       "         -0.75070607, -0.75342852, -0.75288648, -0.75542863, -0.75739083,\n",
       "         -0.75431802, -0.7587627 , -0.75773487, -0.75951558, -0.76002421,\n",
       "         -0.76307827, -0.76299727, -0.76452628, -0.76502531, -0.76458657,\n",
       "         -0.76740118, -0.76813063, -0.76862618, -0.76806422],\n",
       "        [-0.52622779, -0.52637462, -0.52887311, -0.52537936, -0.51948361,\n",
       "         -0.51993186, -0.51801452, -0.51945112, -0.51881475, -0.51991014,\n",
       "         -0.51691235, -0.51612315, -0.5184615 , -0.51930929, -0.51993545,\n",
       "         -0.51811201, -0.52133818, -0.52230406, -0.5229399 , -0.5248918 ,\n",
       "         -0.52665893, -0.52827195, -0.52765708, -0.53232165, -0.53457332,\n",
       "         -0.53661386, -0.53620738, -0.54018396, -0.54177437, -0.54348813,\n",
       "         -0.54497758, -0.54315754, -0.54698888, -0.54932254, -0.5505417 ,\n",
       "         -0.54879746, -0.55062466, -0.5534132 , -0.55552259, -0.55641518,\n",
       "         -0.55756244, -0.55859182, -0.55575871, -0.56044314, -0.55915243,\n",
       "         -0.55525835, -0.56307651, -0.56392391, -0.56469619, -0.56552232,\n",
       "         -0.5663013 , -0.56697276, -0.56770749, -0.56786092, -0.56251866,\n",
       "         -0.56802148, -0.56882384, -0.56970633, -0.56565696, -0.57216918,\n",
       "         -0.5726865 , -0.57191342, -0.57380673, -0.56909837, -0.57206409,\n",
       "         -0.57534151, -0.57584271, -0.56923307, -0.57685682, -0.57639016,\n",
       "         -0.57765955, -0.57482186, -0.57852841, -0.57560655, -0.57931017,\n",
       "         -0.57967251, -0.56671314, -0.57495399, -0.57492454, -0.58117381,\n",
       "         -0.58035435, -0.57872015, -0.58098165, -0.58242312, -0.58138457,\n",
       "         -0.58316756, -0.57481707, -0.5836073 , -0.58285988, -0.58419296,\n",
       "         -0.5835433 , -0.58372766, -0.58410525, -0.58434173, -0.58552318,\n",
       "         -0.58585613, -0.586062  , -0.58238366, -0.58655709]]),\n",
       " 2: array([[-0.54736825, -0.55054212, -0.54835156, -0.53503031, -0.53442642,\n",
       "         -0.52846114, -0.53084237, -0.52324181, -0.52808641, -0.52687706,\n",
       "         -0.52698107, -0.52731054, -0.52788452, -0.52852774, -0.53265308,\n",
       "         -0.53605012, -0.53767303, -0.54080516, -0.54109536, -0.54364507,\n",
       "         -0.54266634, -0.55155018, -0.5514325 , -0.55513294, -0.54664835,\n",
       "         -0.54928365, -0.55324855, -0.54933765, -0.54761341, -0.5506218 ,\n",
       "         -0.55072203, -0.55699092, -0.55336839, -0.5505938 , -0.52401802,\n",
       "         -0.55245168, -0.55892086, -0.53879728, -0.53485747, -0.55268248,\n",
       "         -0.55327462, -0.55198393, -0.5581648 , -0.55052919, -0.53483609,\n",
       "         -0.54861374, -0.54867353, -0.553211  , -0.55419298, -0.55121761,\n",
       "         -0.54387029, -0.54359966, -0.54469659, -0.54620931, -0.52896412,\n",
       "         -0.54418509, -0.55006668, -0.53932723, -0.54450451, -0.53967722,\n",
       "         -0.54184584, -0.53945426, -0.5292587 , -0.53884011, -0.51930001,\n",
       "         -0.52009773, -0.52142129, -0.53631043, -0.54947358, -0.5454876 ,\n",
       "         -0.53875632, -0.54383127, -0.54319768, -0.52631689, -0.53869818,\n",
       "         -0.55390685, -0.5452034 , -0.54746062, -0.5409594 , -0.54454573,\n",
       "         -0.54052647, -0.55002186, -0.53856035, -0.54300221, -0.54474463,\n",
       "         -0.5397333 , -0.54896931, -0.55297411, -0.55154876, -0.55141873,\n",
       "         -0.55345563, -0.5356768 , -0.54535932, -0.54717952, -0.54699615,\n",
       "         -0.55231702, -0.5402251 , -0.54834686, -0.55404495],\n",
       "        [-0.5837316 , -0.58759109, -0.5818815 , -0.57664928, -0.57545035,\n",
       "         -0.57640252, -0.57986868, -0.58603896, -0.59088204, -0.59712619,\n",
       "         -0.60330036, -0.60968005, -0.6126267 , -0.61969891, -0.62577803,\n",
       "         -0.6308308 , -0.63241248, -0.64061725, -0.64245797, -0.64840519,\n",
       "         -0.65171361, -0.65395986, -0.65801632, -0.65947536, -0.66289502,\n",
       "         -0.65233485, -0.66844411, -0.66964125, -0.6715525 , -0.67589365,\n",
       "         -0.67827117, -0.6804705 , -0.67202193, -0.6844902 , -0.68746101,\n",
       "         -0.68846575, -0.69032486, -0.69208307, -0.69374712, -0.69569068,\n",
       "         -0.69747282, -0.69697962, -0.69990786, -0.68963871, -0.7034377 ,\n",
       "         -0.70503004, -0.70547721, -0.69660556, -0.70857987, -0.70829471,\n",
       "         -0.70955236, -0.68986674, -0.70924182, -0.71312522, -0.71360934,\n",
       "         -0.69906052, -0.71528147, -0.71578643, -0.71652999, -0.71692537,\n",
       "         -0.7177079 , -0.71634644, -0.71702546, -0.69808312, -0.67862062,\n",
       "         -0.72090082, -0.7073767 , -0.7220916 , -0.72094865, -0.72316467,\n",
       "         -0.72490139, -0.72461843, -0.72272317, -0.72382702, -0.72430747,\n",
       "         -0.72663572, -0.72721502, -0.7270253 , -0.70733425, -0.72850309,\n",
       "         -0.72954571, -0.7109147 , -0.7282333 , -0.72905531, -0.7301247 ,\n",
       "         -0.7308795 , -0.7183667 , -0.73089971, -0.71563725, -0.73159933,\n",
       "         -0.73392993, -0.73300761, -0.73445948, -0.73371452, -0.7245352 ,\n",
       "         -0.73378604, -0.73608891, -0.72493929, -0.73647939],\n",
       "        [-0.55075367, -0.55345709, -0.57323046, -0.57732286, -0.57858781,\n",
       "         -0.58071769, -0.58274775, -0.58770736, -0.58585242, -0.59634352,\n",
       "         -0.60293952, -0.61379475, -0.6191365 , -0.62120502, -0.6365229 ,\n",
       "         -0.63325507, -0.63904907, -0.64694096, -0.65180141, -0.64129496,\n",
       "         -0.66194259, -0.67301459, -0.67244792, -0.66692943, -0.67972087,\n",
       "         -0.68621521, -0.69035931, -0.69914708, -0.70185047, -0.70312131,\n",
       "         -0.70657753, -0.71114452, -0.71308742, -0.71905903, -0.71803973,\n",
       "         -0.72290557, -0.72638616, -0.72829128, -0.73273754, -0.72074717,\n",
       "         -0.73744535, -0.73755394, -0.73789576, -0.74001393, -0.7446576 ,\n",
       "         -0.74705774, -0.75080151, -0.75437484, -0.74749445, -0.75392111,\n",
       "         -0.749573  , -0.75010675, -0.75539203, -0.75200409, -0.75783492,\n",
       "         -0.75137668, -0.75788001, -0.75768334, -0.75140909, -0.76668826,\n",
       "         -0.77384635, -0.7706657 , -0.76863513, -0.76883457, -0.77263967,\n",
       "         -0.75984282, -0.77418242, -0.77217891, -0.77263858, -0.76650951,\n",
       "         -0.77159526, -0.76586727, -0.76137703, -0.77565401, -0.77251028,\n",
       "         -0.78749527, -0.77482672, -0.78148153, -0.77754764, -0.78202667,\n",
       "         -0.7816735 , -0.77743722, -0.76492439, -0.78877909, -0.76662598,\n",
       "         -0.78134712, -0.78638309, -0.77716953, -0.77923565, -0.80478364,\n",
       "         -0.79461205, -0.79400014, -0.79430219, -0.79530703, -0.79822393,\n",
       "         -0.79629927, -0.79130752, -0.80641981, -0.79234446],\n",
       "        [-0.52540553, -0.52599981, -0.52855272, -0.52617706, -0.52045277,\n",
       "         -0.51776349, -0.5202662 , -0.52033207, -0.5212414 , -0.52282865,\n",
       "         -0.52469581, -0.52646645, -0.52921016, -0.5306997 , -0.53377731,\n",
       "         -0.5354889 , -0.5377007 , -0.53852441, -0.53947845, -0.5408327 ,\n",
       "         -0.54061805, -0.54146076, -0.5415987 , -0.54143714, -0.54215387,\n",
       "         -0.54265286, -0.5431347 , -0.54360077, -0.54300666, -0.54313541,\n",
       "         -0.54538392, -0.54626308, -0.54472812, -0.5471695 , -0.54757871,\n",
       "         -0.54825473, -0.54629942, -0.5491141 , -0.54930532, -0.54917974,\n",
       "         -0.54937009, -0.54961466, -0.55035925, -0.55011996, -0.55050824,\n",
       "         -0.55117563, -0.54963831, -0.55173132, -0.55195285, -0.54993507,\n",
       "         -0.55226625, -0.55268475, -0.55254571, -0.54940653, -0.5522347 ,\n",
       "         -0.55015061, -0.55208173, -0.55205524, -0.55206606, -0.55174141,\n",
       "         -0.5517521 , -0.55090835, -0.5483275 , -0.5506446 , -0.55140679,\n",
       "         -0.55132916, -0.55147327, -0.55167536, -0.5501991 , -0.55191564,\n",
       "         -0.55178857, -0.55138872, -0.55115593, -0.55129249, -0.55058175,\n",
       "         -0.55182542, -0.55204874, -0.55141347, -0.55017727, -0.55203126,\n",
       "         -0.55037563, -0.55240006, -0.55223985, -0.55162289, -0.54821193,\n",
       "         -0.54890285, -0.55197929, -0.55210544, -0.55317506, -0.55198354,\n",
       "         -0.5522127 , -0.55258907, -0.55201156, -0.55172038, -0.5506939 ,\n",
       "         -0.55318297, -0.55352521, -0.55350906, -0.5524328 ],\n",
       "        [-0.59073683, -0.59287959, -0.58840026, -0.58390421, -0.58481914,\n",
       "         -0.58270774, -0.582699  , -0.58113958, -0.58046765, -0.58201541,\n",
       "         -0.58092645, -0.58909944, -0.59347652, -0.59795278, -0.60257134,\n",
       "         -0.60552797, -0.60721336, -0.61061262, -0.60552956, -0.60834711,\n",
       "         -0.61242441, -0.6177805 , -0.62056246, -0.61374284, -0.62380428,\n",
       "         -0.62536339, -0.61783273, -0.62761077, -0.62884459, -0.62599053,\n",
       "         -0.63089945, -0.62481212, -0.63269803, -0.63405661, -0.63513205,\n",
       "         -0.63628222, -0.63743961, -0.63874998, -0.64021673, -0.641309  ,\n",
       "         -0.63038981, -0.64109892, -0.64517365, -0.64577025, -0.63912272,\n",
       "         -0.6329974 , -0.63996408, -0.64777693, -0.63481205, -0.64969383,\n",
       "         -0.65248148, -0.6415681 , -0.65392406, -0.65521387, -0.64737811,\n",
       "         -0.65602079, -0.65663586, -0.65724388, -0.65694787, -0.64484569,\n",
       "         -0.65969013, -0.65838923, -0.66014848, -0.66053519, -0.64899099,\n",
       "         -0.66063179, -0.66203572, -0.663016  , -0.663283  , -0.66266801,\n",
       "         -0.66559336, -0.66468954, -0.65442632, -0.66677996, -0.66623248,\n",
       "         -0.66623002, -0.66421595, -0.66739781, -0.66862669, -0.658371  ,\n",
       "         -0.66131318, -0.66881047, -0.65517732, -0.67082396, -0.66999673,\n",
       "         -0.6703559 , -0.65006774, -0.67106661, -0.6719858 , -0.6659432 ,\n",
       "         -0.67187955, -0.67188383, -0.67334501, -0.66880111, -0.67363305,\n",
       "         -0.6734215 , -0.67499114, -0.66979066, -0.6684099 ]])}"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lrcv_L1.scores_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[-0.53587315 -0.53592805 -0.53775672 -0.53818725 -0.53646738 -0.53762102\n",
      " -0.53955635 -0.54208001 -0.54491982 -0.54794139 -0.54850736 -0.55312289\n",
      " -0.55630331 -0.56074496 -0.56184847 -0.5615772  -0.56802398 -0.57001991\n",
      " -0.57249087 -0.57623672 -0.57795046 -0.57882736 -0.58282645 -0.58318922\n",
      " -0.58582839 -0.58892781 -0.58913027 -0.59223611 -0.59577506 -0.59702053\n",
      " -0.59977728 -0.60015241 -0.60342762 -0.60579222 -0.60296711 -0.60201861\n",
      " -0.60711443 -0.60125424 -0.60768051 -0.61014637 -0.61164588 -0.61591159\n",
      " -0.60692091 -0.61802328 -0.61489024 -0.61505718 -0.61884914 -0.61368159\n",
      " -0.62207889 -0.62286797 -0.62365786 -0.61757098 -0.62618018 -0.62678724\n",
      " -0.62460699 -0.62818088 -0.62950039 -0.63190529 -0.63252918 -0.62920895\n",
      " -0.63444525 -0.63518352 -0.63398123 -0.62925992 -0.63645749 -0.63561383\n",
      " -0.6356459  -0.63900224 -0.63497182 -0.64365008 -0.64257358 -0.63859592\n",
      " -0.63692171 -0.63874983 -0.64190614 -0.64178901 -0.64461534 -0.63737571\n",
      " -0.64350195 -0.64418454 -0.64845267 -0.64607548 -0.6483782  -0.64209254\n",
      " -0.64851883 -0.64619539 -0.64683539 -0.64331244 -0.65191041 -0.64852056\n",
      " -0.64717535 -0.65441917 -0.65562569 -0.65134975 -0.64803643 -0.65889749\n",
      " -0.64860944 -0.64788246 -0.65091404]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZQAAAEKCAYAAAA1qaOTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Wd4XeWZ9v3/pW5LtixbsiUjC+OC\njY0biGp6J4SSAgRCQvNLCGGGDAntZZ4hTOZJSEjITEKSobckdAgmdGw6MSCDG+69yL3L6tL1fFhL\nRhjJ2pL31lI5f8exj732qqeW5X1ptfs2d0dERGRfJUUdQEREugYVFBERiQsVFBERiQsVFBERiQsV\nFBERiQsVFBERiQsVFBERiQsVFBERiQsVFBERiYuUqAO0p9zcXB88eHDUMUREOpXp06dvcve8lubr\nVgVl8ODBlJSURB1DRKRTMbMVscynU14iIhIXKigiIhIXKigiIhIXKigiIhIXKigiIhIXKigiIhIX\nKigiIhIX3eo5FOm66uqdRRt2Mnv1dipr6qird+od6t3x8L3hc32jaQBjC7M5Zngu6SnJEf8UIp2b\nCop0Slt3VTNj1TY+XbmVT1duZeaq7ZRV1bZ5fVnpKZw0sj9nHpzP8SPy6Jmm/xoiraX/NdLh1dU7\nC9bt5NOVW/ls5TY+W7mVpZt2AZBkMDK/N+dNGMghRTmMG9SH7B6pJJmRZGDhe3KSkWSGGeG0YHxN\nnfPBkk28Onsdr89dx+SZpWSkJnHCgf05c0w+J43sT6+M1Ij3gEjnYB4e9ncHxcXF3p2aXqmvdyz8\nUu0s3J0NO6uYs2Z7cPSxYhszV2+jvLoOgH6ZaUwoyuGQ/fswYVAOYwuzyUyPz99FtXX1fLxsC69+\nvo5X56xjw84q0pKTOGZ4LmccnM9powbQp2daXLYl0pmY2XR3L25xPhWU9lFbV09lbT1VNXVU1tZT\nWVMXvuqpqq2jqiYc96Xh4L1q9/zB8rs/1wbjmpunuq6etOQk+mWl0S8rjdys9EavLz43TOubmUZy\nUuKLT1lVLau2lLNqSzkrw/dVWytYuaWc1VvLqaypB4KjilEFvTmkqE9QRIpyGNS3R7sUyPp657NV\nW3ll9jpembOONdsqSEkyjjswj3PGDeTUUQPiVshEOroOXVDMrC/wJDAYWA5c4O5bm5m3NzAPeN7d\nrw3HpQF3AycA9cCt7v5sS9tta0F5dc46lm4qC77oG3/h7/FFvntao3kaptfWt30/pyUnkZ6aRHpK\nMhmpSWSkBu+7P6ckk5GaTHpKEumpX8yTnpJEZU09m8uq2FRWxaayajaVVbG5rJrquvqvbCfJoG9m\nGv0y08ntFRSZxsN5jYpPv6y0Zi9i19bVs3Z75e5isbJRwVi1pZwtu6q/NH+v9BQG9e3JoL49KOrb\nk0F9ezIyvzdj9sumR1r0F8rdnTlrdvDirFJenFnK2u2VZKQmcfJBAzhn3EBOGJGnC/rSpXX0gvJr\nYIu732FmNwM57n5TM/P+D5AXzt9QUG4Hkt39380sCejr7pta2m5bC8oVD3/C1PkbAEhP+eLLOmOP\nL+/gfY8v+9QvvvTTGz6Hw1+eZ491Npon3kcN7s6OytrdxWVTQ8HZWcWmXdXBe1kVm8PhXeHppj31\nykghLzzK6ZuZxs6qGlZuKad0WyV1jQpoSpKxX05QLApzelLUt2dYOIJx2T1SO81pufp6p2TFVibP\nXMPLs9exZVc1vTJSOGN0PueMH8jRQ3Pb5ShPpD119IKyADjB3deaWQHwtruPaGK+Q4EbgFeB4kYF\nZRUw0t13tWa7bS0o5dW1JJmRnpLUab744qmiuu6LorP7KCcY3hgWos3hF2tR354MCotGYVgw8ntn\nkJLc9R55qqmr54PFm3hx5lpe+3wdZVW1FOb04PtH7c+FxUVk99TFfOkaOnpB2ebufRp93uruOXvM\nkwRMBb4HnExYUMysDzAbeJrglNcS4Fp3X9/SdrvbRXlpP5U1dUyZt4HHpi1n2tItZKQm8Y0JhVx2\n9GBG5PeKOp7IPom1oCTsqqKZvQnkNzHp1hhXcQ3wsruv2uOoIAUoBD5w9+vN7HrgNwSFp6kcVwFX\nARQVFcW4aZHWyUhN5qyxBZw1toB5a3fwyIfLee7T1Tz+8UqOGtKPyyYO5pSDBuh0mHRpHfaUl5n9\nFTiW4KJ7FpAG/Am4BSgDerl7vZkNAl5199EtbVdHKNKetu6q5smSVTz2zxWs2VZBYU4Pbj9nNCcf\nNCDqaCKtEusRSlQnticDl4bDlwIv7DmDu3/X3YvcfTDwU+BRd7/Zgwr4IsHpLghOh81NeGKRVsrJ\nTOPq44fyzg0n8L+XHEpWegpXPlLCzyZ/TlVt0zc6iHRmURWUO4BTzWwRcGr4GTMrNrP7Y1j+JuBn\nZjaL4FTXTxKWVGQfpSQnccbB+bxw7UQunziYhz9czjf++CFLNpZFHU0krvRgo0g7mzJvPT99eiaV\nNfXcfu5ozj+0sFvePSidR0c/5SXSbZ180ABeue44xg3K5sZnZnHdEzPYUVkTdSyRfaaCIhKB/OwM\n/jrpSH562oG8NHstZ/3+Pd5f1OKzuSIdmgqKSESSk4xrTxrOUz84EsO45IGPuOav01mzrSLqaCJt\nooIiErFD9+/L6/92HD859UCmzt/Ayb99m7unLqKyRneCSeeigiLSAWSkJvMvJw/nzeuP58QR/fnN\n6ws5/b/fZer8FhuAEOkwVFBEOpDCnJ78+ZJDeezKw0lJMq54uIT/eGEO3eluTOm8VFBEOqBjh+fx\nynXHccXEA3j0nyu4/71lUUcSaZF6CBLpoNJSkvj3sw5i3Y4KfvHKPIr69eT00U01jyfSMegIRaQD\nS0oy7rpgPGML+/DjJ2YwZ832qCOJNEsFRaSDy0hN5r7vH0rfzDSufOQT1m7XbcXSMamgiHQC/Xtl\n8MBlxeyqquPKh0vYVVUbdSSRr1BBEekkRub35g8XT2D+uh1c98SML3WzLNIRqKCIdCInjujPbWeP\n5s156/nJUzOoqauPOpLIbrrLS6STufTowZRV1XLnawvYXlHDn757KD3SkqOOJaIjFJHO6EcnDuMX\n3xjDOws3cskDH7GtvDrqSCIqKCKd1cVHFPHHiw9h9urtXHjPNNZtr4w6knRzKigindiZYwp4+PLD\nWL21nG/9+UOWbdoVdSTpxlRQRDq5o4fl8sRVR1FRU8dF905j1ZbyqCNJN6WCItIFjCnM5q+TjqC8\nupbvPfARG3bq9Je0PxUUkS7ioILePHT54azfUcX3H/iY7eXqVljalwqKSBdy6P453Pv9Q1m6cRdX\nPPIJ5dV6ol7ajwqKSBdz7PA8fn/ReD5buZWr//Ip1bV6+FHahwqKSBd0xsEF3PGtsby7cCM3PTtL\nHXRJu9CT8iJd1AXFg1i/vZLfvrGQ0QN7M+nYIVFHki5ORygiXdi1Jw3jjNH5/OLleXyweFPUcaSL\nU0ER6cLMjN9cMI5h/bO49m+f6hkVSSgVFJEuLis9hXu/V0xdvXPVY9N155ckjAqKSDcwODeT318U\n9KVywzOzqKqtizqSdEEqKCLdxAkj+nPj6SN5adZaTrjzbR7+YBmVNSosEj8qKCLdyNXHD+HRKw6n\nMKcHP3txLsf86i3ufXeJjlgkLnTbsEg3YmYcd2Aexx2Yx0dLN3P3W4v5xcvzKd1Wyc/OGR11POnk\nIjtCMbO+ZvaGmS0K33P2Mm9vM1tjZnc3GneRmc02s1lm9qqZ5bZPcpGu4Ygh/XjsyiO4+Igi/jJt\nBUs2lkUdSTq5KE953QxMcffhwJTwc3N+DrzT8MHMUoD/AU5097HALODaBGYV6bKuP/VAMlKT+eXL\n86OOIp1clAXlXOCRcPgR4LymZjKzQ4EBwOuNR4evTDMzoDdQmrioIl1XblY6PzpxGG/OW8+HevhR\n9kGUBWWAu68FCN/77zmDmSUBvwVuaDze3WuAHwKzCQrJKOCBpjZiZleZWYmZlWzcuDG+P4FIF3H5\nxMHs16cHP39pHnX1avdL2iahBcXM3jSzOU28zo1xFdcAL7v7qj3Wm0pQUCYAAwlOed3S1Arc/V53\nL3b34ry8vH34aUS6rozUZG4+cyTz1u7g2emro44jnVRC7/Jy91Oam2Zm682swN3XmlkBsKGJ2Y4C\njjWza4AsIM3MyoBnw/UvCdf1FHu/BiMiLfj62AIe+mAZd76+gLPGFpCZrptApXWiPOU1Gbg0HL4U\neGHPGdz9u+5e5O6DgZ8Cj7r7zcAaYJSZNRxynArMS3xkka7LzPg/Xx/Fxp1V/H7qoqjjSCcUZUG5\nAzjVzBYRFIQ7AMys2Mzu39uC7l4K3A68a2azgPHALxKcV6TLm1CUw4XFg7j/vWXMWbM96jjSyVh3\n6ninuLjYS0pKoo4h0qFtL6/h5LveoSA7g+evOZqUZDWo0d2Z2XR3L25pPv2miMiXZPdM5fZzRjN7\nzXYe+mB51HGkE1FBEZGv+NqYfE45aAC/fWMBKzerDxWJjQqKiHyFmfHz80aTkpTE///8bPVJLzFR\nQRGRJhVk9+CmM0fy/uJN/PjJGVRUq0Vi2TvdaC4izbrkiCK27armrjcXsmDdTu753qHs3y8z6ljS\nQekIRUSaZWb8y8nDeeiyw1i7vZKz//A+U+evjzqWdFAqKCLSohNG9Ocf/3IMhTk9ueLhEn7+j7nq\nlEu+QgVFRGIyqG9PnrvmaC49an8eeH8Z3/zTh+pDRb5EBUVEYpaRmszt5x7Mfd8vpnRbBWf/4X0W\nrt8ZdSzpIFRQRKTVTh01gJevO5bkJONXr6hjLgmooIhImxRk9+CaE4YxZf4GPlq6Oeo40gGooIhI\nm10+cTD5vTP45Svz9fCjqKCISNtlpCbzb6cOZ8aqbbz2+bqo40jEVFBEZJ9865BChvfP4tevLqCm\nrj7qOBIhFRQR2ScpyUnceMZIlm7axVMlq1peQLosFRQR2WenHNSfQ/fP4Q9TFlNZowceuysVFBHZ\nZ2bGDaePYN2OSv4ybUXUcSQiKigiEhdHDunHscNz+eNbi9lZWRN1HImACoqIxM0Np49ga3kND76/\nPOooEgEVFBGJm7GFfThjdD73vbeUrbuqo44j7SymgmJmE80sMxy+xMzuMrP9ExtNRDqjn5x2ILuq\na7lt8ufU1+thx+4k1iOUPwPlZjYOuBFYATyasFQi0mkNH9CLn542gskzS/nVq2rnqzuJtcfGWnd3\nMzsX+B93f8DMLk1kMBHpvK45YSjrd1Ryz7tLyeuVzoWHDeKT5VuYv24n3z9qMFnp6iy2K4r1X3Wn\nmd0CXAIcZ2bJQGriYolIZ2Zm3Hb2aDburOK/XprHL1+ZT114+qu+3rn2pOERJ5REiLWgXAhcDFzp\n7uvMrAi4M3GxRKSzS04yfnfheAb0nk+vjBSOGtKPP769mMemreAHxw8lNVn3BHU1MR+hEJzqqjOz\nA4GRwOOJiyUiXUFGajI/O2f07s8VNXVc+UgJr85Zx9njBkaYTBIh1j8R3gXSzWw/YApwOfBwokKJ\nSNd04oj+7N+vJ498uDzqKJIAsRYUc/dy4JvAH9z9G8DoFpYREfmSpCTje0fuT8mKrcxZsz3qOBJn\nMRcUMzsK+C7wUjguOTGRRKQrO794ED3TknlYRyldTqwF5cfALcDz7v65mQ0B3kpcLBHpqrJ7pPKt\nQwr5+2dreGfhxqjjSBzFVFDc/R13Pwf4k5lluftSd//Xtm7UzPqa2Rtmtih8z2lmvjozmxG+Jjca\nf4CZfRQu/6SZpbU1i4i0v5+ePoIDB/TiB4+V8MnyLVHHkTiJtemVMWb2GTAHmGtm081sX66h3AxM\ncffhBBf5b25mvgp3Hx++zmk0/lfA78LltwJX7kMWEWln2T1SefTKwxmY3YMrHvpE11O6iFhPed0D\nXO/u+7t7EfAT4L592O65wCPh8CPAebEuaGYGnAQ805blRaRjyM1K5y+TjiArI4Vr//Yp5dW1UUeS\nfRRrQcl0993XTNz9bSBzH7Y7wN3XhutaC/RvZr4MMysxs2lm1lA0+gHb3L3ht281sN8+ZBGRiAzs\n04O7LhjP8s3l/OoVtfvV2cX6YONSM/s/wGPh50uAZXtbwMzeBPKbmHRr7PEocvfS8CaAqWY2G9jR\nxHzNNmlqZlcBVwEUFRW1YtMi0h6OGtqPyycO5qEPlnP66HyOHpYbdSRpo1iPUK4A8oDngOfD4cv3\ntoC7n+LuBzfxegFYb2YFAOH7hmbWURq+LwXeBiYAm4A+ZtZQDAuB0r3kuNfdi929OC8vL8YfV0Ta\n042nj2RIbiY3PDOLimr1Sd9ZxXqX11Z3/1d3P8TdJ7j7de6+dR+2OxloaK34UuCFPWcwsxwzSw+H\nc4GJwFx3d4Jblr+9t+VFpPPokZbMf33jYNZsq+DZT1dHHUfaaK+nvMzsRfZyOmmPO69a4w7gKTO7\nElgJnB9urxi42t0nAQcB95hZPUHhu8Pd54bL3wQ8YWb/BXwGPNDGHCLSQRw1pB/jCrN54P1lXHR4\nEclJFnUkaaWWrqH8JhEbdffNwMlNjC8BJoXDHwJjmll+KXB4IrKJSDTMjP/vuCFc+7fPeHPeek4f\n3dQlWOnI9lpQ3P2d9goiInLG6HwKc3pw/3tLVVA6oVgfbJxtZrP2eL1nZr8zs36JDiki3UNKchJX\nHnMAnyzfSomeoO90Yr3L6xWCRiG/G75eBN4D1qFm7EUkji4oHkRuVjqXP/wJr3++Luo40gqxFpSJ\n7n6Lu88OX7cCx7v7r4DBiYsnIt1NZnoKz19zNIP7ZXLVY9O5e+qiqCNJjGItKFlmdkTDBzM7HMgK\nP6q9BBGJq0F9e/L01Udx1pgCfvfmIlZs3hV1JIlBrAVlEnC/mS0zs+XA/cAkM8sEfpmocCLSfWWk\nJnPb2aNISTLunro46jgSg1gfbPzE3ccA44Hx7j42HLfL3Z9KbEQR6a76987gosOLeO6zNazcXB51\nHGlBrHd5ZZvZXQRNzb9pZr81s+zERhMRgR+eMJTkJOPut3QtpaOL9ZTXg8BO4ILwtQN4KFGhREQa\nDOidwcWHF/Hsp2vUb0oHF2tBGerut4U9NS5199uBIYkMJiLS4LqTh9MvM40fPzmDyho1HtlRxVpQ\nKszsmIYPZjYRqEhMJBGRL8vJTOO3F4xj8YYyfvnyvKjjSDNi7Q/lh8Aj4XUTA7YAlyUqlIjIno4d\nnscVEw/gwQ+WMbKgNxcdrv6NOpqYCoq7zwDGmVnv8HNTnVyJiCTUjWeMYOmmMm55bjbl1XVcecwB\nUUeSRlpqvv76ZsYD4O53JSCTiEiTMlKTued7h3Ld4zP4+T/m0i8zjfMmqAfwjqKlayi9WniJiLSr\n9JRk7r54AhOK+nD7i5+zuawq6kgSaqn5+tvbK4iISKxSkpO445tj+fof3uO/XprH7y4cD8D28hp+\n/tJcbjx9BP17Z0ScsvuJ9S6v3czs00QEERFpjRH5vbj6+KE8/9kaPly8CYCnSlbxzPTVPD1d3QhH\nodUFheAuLxGRyP3oxGHk987g91MX4e48PX0VAK/PXR9xsu6pLQXlpbinEBFpg4zUZCYdewDTlm7h\n0X+uYOH6MobkZTJz1TbWba+MOl630+qC4u7/noggIiJtcdHhRfTpmcrtL35OekoSd357HABvzNNR\nSnuLtXHInWa2Y4/XKjN73szUBIuIRCYzPYVLjxpMvcMZB+dzSFEfDsjNVG+PEYj1Sfm7gFLgbwTX\nUL4D5AMLCBqOPCER4UREYnHZ0YP5eNkWJh0zBDPjtFEDePCDZWyvqCG7RyqTZ5aSkmR8bUxB1FG7\ntFhPeZ3h7ve4+0533+Hu9wJfc/cngZwE5hMRaVFOZhqPX3UkYwqDXjXOGltATZ3z4sxSKqrruPW5\n2dzw9Ey27qqOOGnXFmtBqTezC8wsKXxd0GiaJyKYiEhbjdkvm5H5vXiqZBWvzFnLzqpadlXXcd97\nS6OO1qXFWlC+C3wP2ACsD4cvMbMewLUJyiYi0iZmxncOG8Ss1dv53ZsLOSA3k7PGFvDIh8vZoqOU\nhIm1C+Cl7n62u+e6e144vNjdK9z9/USHFBFprfMm7EdaShKrtlRwfnEh/3bKcHZV1/H4xyujjtZl\nxXqX14FmNsXM5oSfx5qZbh8WkQ6rT880zhidT3KS8e1DChnWvxcTivrwWnj311sLNvDW/A0Rp+xa\nYj3ldR9wC1AD4O6zCO70EhHpsP7j7FE8cdWRu9v1Om1UPrNWb2fZpl1c/+QMfqHOuuIq1oLS090/\n3mNcbbzDiIjEU25WOocN7rv786mjBgDw4yc+Y2t5Dcs376Kmrj6qeF1OrAVlk5kNJbyjy8y+DaxN\nWCoRkQQY1j8raJpl9XaSDGrqnBWby6OO1WXEWlB+BNwDjDSzNcCPgasTlkpEJEFOG5UPwCVH7g/A\n4g1lUcbpUmJ9Un4N8BDwFtAX2AFcCvxnWzZqZn2BJ4HBwHLgAnff2sR8dcDs8ONKdz8nHP9XoJjg\nms7HwA/cvaYtWUSke7n06P0xg6uOHcKj/1zBko0qKPES6xHKC8DZBF/gpUAZsGsftnszMMXdhwNT\nws9NqXD38eHrnEbj/wqMBMYAPYBJ+5BFRLqRguwe3HTGSHIy0xiYncHiDWVU19azvUJ/k+6rWI9Q\nCt39jDhu91y+aP/rEeBt4KZYF3b3lxuGzexjoDCO2USkmxjaP4tFG3Zy2+TPmTJvPe/ddCLpKclR\nx+q0Yj1C+dDMxsRxuwPcfS1A+N6/mfkyzKzEzKaZ2Xl7TjSzVIKn9l+NYzYR6SaG9c9i0foynpm+\nig07q3h1jloo3hexHqEcA1xmZsuAKoIWh93dxza3gJm9SdAi8Z5ubUW+IncvDZvIn2pms919SaPp\nfwLedff39pLjKuAqgKKiolZsWkS6umH9s6iqrccsuMX48Y9Xcu74/aKO1WnFWlDObO2K3f2U5qaZ\n2XozK3D3tWZWQNBGWFPrKA3fl5rZ28AEYEm4jtuAPOAHLeS4F7gXoLi4WA1Zishuw/v3AuC0UQMY\nW9iHO19bwPx1OxiZ3zviZJ1TrG15rWjqtQ/bnUxwlxjh+wt7zmBmOWaWHg7nAhOBueHnScDpwEXu\nrqeSRKRNxuyXzZkH53P9qSM4v7iQnmnJnHP3B/z61flRR+uUzL39/2g3s37AU0ARsBI43923mFkx\ncLW7TzKzowmefaknKHz/7e4PhMvXAiuAneEqn3P3Fm9hLi4u9pKSkvj/QCLSJazYvIs7X1vAP2at\n5S9XHkFer3QqauoYP6hP1NEiZWbT3b24xfmiKChRUUERkZZU1dZx8m/fITnJ2LizCnd4+bpjOSA3\nM+pokYm1oMR6l5eISLeQnpLMDaePYMXmcvKzM0hNNn769Ezq6r/8x/eqLeVU1tRFlLJjivWivIhI\nt3H22IGYGUcP7ceUeeu56dnZTF+xlcMPCBqarK2r58z/eY9rThzKNScMizhtx6EjFBGRPSQlGeeM\nG0huVjonjggek/u8dPvu6et3VlFWVctKNSz5JSooIiJ7kdcrndysNOaW7tg9rnRbBQAbd1ZFFatD\nUkEREdkLM+Oggt7MW9dEQSlTQWlMBUVEpAWjCnqzcF3Z7s641uxxhOLufLJ8C93prtmmqKCIiLTg\noILeVNfVs3Rj0Mh6wxHKprIq6uud9xZt4vz//SczVm2LMmbkVFBERFowamDQFMvctcGF+dJtlUDQ\n4+P2iprdfap0994fVVBERFowJDeT9JQkpq8I+gEs3VZBkgXTNpZVsWpLcMRSur0iqogdggqKiEgL\nUpKTOPPgfJ77dA3byqsp3VbBgQOChiU37qxi1dbgyKThVFh3pYIiIhKDq08YSnl1HX96ewk7KmsZ\nVxi077VxZxWrtjQUlMooI0ZOBUVEJAYj83tzykH9uf+9pQCMLwoKyoadlY0Kio5QREQkBnd8ayzH\nH5gHwISiPmSkJrFwfRm7qutISbJuX1DUlpeISIxys9J58LLD2LCzigG9M8jrlc6nK4ML9WMLs/l0\n5TbKqmrJSu+eX606QhERaQUzY0DvDADystJ3P5tyxJB+AKztxkcpKigiIm30vaP2JzU5uH+4oSXi\nNd24oHTP4zIRkTj4xoRCRub3ZvmmXbtvI169VQVFRETa4KCC3hxU0Jv6eqdXRgpz1+5oeaEuSqe8\nRETiICnJGFfYh5nduD0vFRQRkTgZW5jNgnU7d3cNXFtXz/aKmi/NM2/tDibeMZXZq7c3tYpOTQVF\nRCROxg3qQ2297z7tdc+7SznpN29TH/ZH7+7c9sLnrNlWwWPTlkeYNDFUUERE4qShOZb3F23C3Xln\nwUY276pm/c6gSZZPV27l4+Vb6N8rnZdmraW8ujbKuHGngiIiEif52RkcVNCbu95YyL8+MWN3/ygN\nd37NW7sTgJvPHMmu6jreXbgpsqyJoIIiIhJHT/3gSC4+oogXZ5ZSHfbwuDpsjXjJxjJ6piVz5sEF\nJBnMLe1a11FUUERE4qhXRio/PW0EGalffL2uDvtLWbyhjCF5mfRIS2ZIXlaXu8VYBUVEJM76Zqbx\nw+OHcdqoAfTvlc7yzeVMX7GVJRvKGJqXBQT91DecAusq9GCjiEgCXHfKcAC++acPePbT1Tz76WoA\nhjUUlIG9mTyzlG3l1fTpmRZZznjSEYqISAIV5vT80ufBuZlA8IQ9wNzSrnPaSwVFRCSByqqCW4Mv\nKC6keP8cjgxbJR7VUFC60HUUnfISEUmgiw4vYur8Ddxw+kjyeqXvHp/XK53+vdK71BGKCoqISAKd\nOmoAy+84q8lpowb27lJHKJGc8jKzvmb2hpktCt9zmpmvzsxmhK/JTUz/g5mVJT6xiEj8jR7Ym8Ub\nyna3/dXZRXUN5WZgirsPB6aEn5tS4e7jw9c5jSeYWTHQJ8E5RUQSZvTAbGrrnYXrv7h9eHtFDXPW\ndM4HHqMqKOcCj4TDjwDntWZhM0sG7gRujHMuEZF2M6Eo+Jv4o6Vbdo/7/ZRFfOvPH1JdWx9VrDaL\nqqAMcPe1AOF7/2bmyzCzEjObZmaNi861wOSGdeyNmV0VrqNk48aN+55cRCROCrJ7cOCALN5Z+MV3\nU8mKrVTV1rNi864Ik7VNwi6lDoSwAAAN/ElEQVTKm9mbQH4Tk25txWqK3L3UzIYAU81sNlABnA+c\nEMsK3P1e4F6A4uJib8W2RUQS7rjheTw6bQUV1XUkJcG88K6vxRvKGB52K9xZJKyguPspzU0zs/Vm\nVuDua82sANjQzDpKw/elZvY2MIGgoAwDFpsZQE8zW+zuw+L9M4iIJNrxI/K4//1lTFu6mT49U3c3\nKLlkY+e73yiqU16TgUvD4UuBF/acwcxyzCw9HM4FJgJz3f0ld89398HuPhgoVzERkc7qsMF96ZGa\nzNsLNjAtvJaSlZ7C4g1BQdlcVsXgm19i6vz1UcaMSVTPodwBPGVmVwIrCU5hNdy5dbW7TwIOAu4x\ns3qCwneHu8+NKK+ISEJkpCZz9NB+vLVgI6nJxiFFfchMT2FxeIQyf11wB9if3lrCSSMHRBm1RZEU\nFHffDJzcxPgSYFI4/CEwJoZ1ZcU9oIhIOzphRB5T5gdn/n/9rbGs3FLOn99ZwtZd1WzZVQ1AeXXH\nf1ZFT8qLiETs/OJBrNlWSem2Cr4+roDFG8q4+63FTJm/ga1hQWl4+PGpklXkZqV1yKMVFRQRkYhl\npCZz85kjd38es182BdkZvP75OvbL6QHAxrIq3J0bn5kF0GxzLlFSQRER6WDMjKOH5vLOwo0kBXez\nsrOydnff9B2Vmq8XEemAxuzXm01lVcxcvW33uNfnfnGn1+m/e5dVW8qjiNYsFRQRkQ5oTGE2AGu3\nV3LmwfkkGTw7ffXu6QvW7+SxaSuiitcknfISEemARhVk7x4eN6gPK7eU8/kefafMWLltz8UipSMU\nEZEOqEdaMqeOGsDpowdw+cTBHDa4LwCZacl894gijjigL9NXbmXl5nLO/eMHHaKjLh2hiIh0UPd9\nv3j38CVHFrGjoobR+2Vz5TEH8NKstXy0bAu3/n02M1dt4/nPVjNq4KgI06qgiIh0CsP69+KuC8fv\n/jw4tycA7y3aBEBqcvQnnKJPICIirbZ/v8wvfV6/oyqiJF9QQRER6YSy0r98gmn9jsqIknxBBUVE\npJM7aWT/DlFQdA1FRKST+u3541i9tYLNu6ooWb6l5QUSTAVFRKST+tahhQD88a3F7KispaK6jh5p\nyZHl0SkvEZFObr8+QQOSKyNuikUFRUSkkxuRH/Q9P39dtA83qqCIiHRyQ/OySEkypi3dTHVt0Cf9\n2u0V1NV7u+bQNRQRkU4uLSWJtJQkHv94FVW19eyqquW1z9dz8RFFnDSiP6eMap/OuHSEIiLSBVx4\n2CAAnvt0Da99HjRz/7ePVjLp0RJ2Vta0SwYVFBGRLuA/vj6Ki48oAuDwwX3pm5m2e9rkmaXtkkEF\nRUSkCzAzhvfPAiCvV/qXCspb8ze2SwYVFBGRLmL0wKAPleMPzCOnZ+ru8dOWbqamrj7h21dBERHp\nIg4/oC9TfnI85xcXYmFf9Ecc0JeyqloWrNuZ8O3rLi8RkS5kaF5w2ovwjuHziwfxx+8eQm5WesK3\nrSMUEZEuyMOKMjA7o12KCaigiIh0SZlh8/ZpKe33Na9TXiIiXdAd3xzLY9OWM6Eop922qYIiItIF\n5WdncMPpI9t1mzrlJSIicaGCIiIicRFJQTGzvmb2hpktCt+bPMlnZnVmNiN8TW403szs/5rZQjOb\nZ2b/2n7pRUSkKVEdodwMTHH34cCU8HNTKtx9fPg6p9H4y4BBwEh3Pwh4IqFpRUSkRVEVlHOBR8Lh\nR4DzWrn8D4H/dPd6AHffEMdsIiLSBlEVlAHuvhYgfO/fzHwZZlZiZtPMrHHRGQpcGE57xcyGJzqw\niIjsXcJuGzazN4H8Jibd2orVFLl7qZkNAaaa2Wx3XwKkA5XuXmxm3wQeBI5tJsdVwFUARUVFrfoZ\nREQkdgkrKO5+SnPTzGy9mRW4+1ozKwCaPGXl7qXh+1IzexuYACwBVgPPhrM9Dzy0lxz3AvcCFBcX\nt29/mCIi3UhUDzZOBi4F7gjfX9hzhvDOr3J3rzKzXGAi8Otw8t+BkwiOTI4HFsay0enTp28ysxVt\nzJwLbGrjsomkXK3TUXNBx82mXK3TFXPtH8tM5t7+f7SbWT/gKaAIWAmc7+5bzKwYuNrdJ5nZ0cA9\nQD3BtZ7/dvcHwuX7AH8Nly8Ll5mZ4Mwl7l6cyG20hXK1TkfNBR03m3K1TnfOFckRirtvBk5uYnwJ\nMCkc/hAY08zy24CzEplRRERaR0/Ki4hIXKigxO7eqAM0Q7lap6Pmgo6bTblap9vmiuQaioiIdD06\nQhERkbhQQWnEzM43s8/NrD6846y5+c4wswVmttjMbm40/gAz+yhs9PJJM0uLU64WG9M0sxMbNaQ5\nw8wqG1oXMLOHzWxZo2nj2ytXOF9zjXxGub/Gm9k/w3/vWWZ2YaNpcd1fzf2+NJqeHv78i8P9MbjR\ntFvC8QvM7PR9ydGGXNeb2dxw/0wxs/0bTWvy37Sdcl1mZhsbbX9So2mXhv/ui8zs0nbO9btGmRaa\n2bZG0xK5vx40sw1mNqeZ6WZmvw9zzzKzQxpNi+/+cne9whdwEDACeBsobmaeZIKHK4cAacBMYFQ4\n7SngO+Hw/wI/jFOuXwM3h8M3A79qYf6+wBagZ/j5YeDbCdhfMeUCypoZH9n+Ag4EhofDA4G1QJ94\n76+9/b40muca4H/D4e8AT4bDo8L504EDwvUkt2OuExv9Dv2wIdfe/k3bKddlwN1NLNsXWBq+54TD\nOe2Va4/5/wV4MNH7K1z3ccAhwJxmpn8NeAUw4Ejgo0TtLx2hNOLu89x9QQuzHQ4sdvel7l5N0NLx\nuWZmBA9bPhPO15ZGL5vT2sY0vw284u7lcdp+c9rcyGfU+8vdF7r7onC4lKC1hrw4bb+xJn9f9pL3\nGeDkcP+cCzzh7lXuvgxYHK6vXXK5+1uNfoemAYVx2vY+5dqL04E33H2Lu28F3gDOiCjXRcDjcdr2\nXrn7uwR/QDbnXOBRD0wD+ljQQknc95cKSuvtB6xq9Hl1OK4fsM3da/cYHw+xNqbZ4Dt89Zf5/4aH\nu78zs/R2ztVUI58dZn+Z2eEEf3UuaTQ6Xvurud+XJucJ98d2gv0Ty7KJzNXYlQR/5TZoruHW9sr1\nrfDf5xkzG9TKZROZi/DU4AHA1EajE7W/YtFc9rjvr27Xp7ztpdFKd/9KEzBNraKJcb6X8fucK9Z1\nhOspIHgg9LVGo28B1hF8ad4L3AT8Zzvm+kojn8COJuaLan89BlzqYXcI7MP+amoTTYzb8+dMyO9U\nC2Jet5ldAhQTNHPUoLmGW9sj14vA4x40y3Q1wdHdSTEum8hcDb4DPOPudY3GJWp/xaLdfr+6XUHx\nvTRaGaPVBJ17NSgESgnayOljZinhX5kN4/c5l8XYmGboAuB5d69ptO614WCVmT0E/LQ9c3nTjXw+\nS8T7y8x6Ay8B/x6eCmhYd5v3VxOa+31pap7VZpYCZBOcwohl2UTmwsxOISjSx7t7VcP4Zv5N4/EF\n2WIuD1raaHAf8KtGy56wx7JvxyFTTLka+Q7wo8YjEri/YtFc9rjvL53yar1PgOEW3KGURvDLM9mD\nq1xvEVy/gGYavWyjhsY0Y1nvV87dhl+qDdctzgOavBskEbnMLKfhlJF90cjn3Kj3V/hv9zzBueWn\n95gWz/3V5O/LXvJ+G5ga7p/JwHcsuAvsAGA48PE+ZGlVLjObQNCe3jneqBO75v5N2zFXQaOP5wDz\nwuHXgNPCfDnAaXz5SD2hucJsIwgucP+z0bhE7q9YTAa+H97tdSSwPfyjKf77K1F3HnTGF/ANgqpd\nBawHXgvHDwRebjTf1whaOF5CcKqsYfwQgv/wi4GngfQ45epH0FXyovC9bzi+GLi/0XyDgTVA0h7L\nTwVmE3wx/gXIaq9cwNHhtmeG71d2hP0FXALUADMavcYnYn819ftCcArtnHA4I/z5F4f7Y0ijZW8N\nl1sAnBnn3/eWcr0Z/j9o2D+TW/o3badcvwQ+D7f/FkFX4A3LXhHux8XA5e2ZK/z8M+COPZZL9P56\nnOAuxRqC768rgasJGs2F4NTWH8Pcs2l0B2u895eelBcRkbjQKS8REYkLFRQREYkLFRQREYkLFRQR\nEYkLFRQREYkLFRSRODKzsn1c/pnwaWrMLMvM7jGzJRa0ivyumR1hZmnhcLd7MFk6NhUUkQ7CzEYT\ntCa8NBx1P8ET88PdfTRBK7u5HjROOAW4sMkViUREBUUkAcKnku80szlmNtvC/lbMLMnM/hQecfzD\nzF42s4bWAr5L+FS/mQ0FjiBoFqYegmY73P2lcN6/h/OLdBg6ZBZJjG8C44FxQC7wiZm9S9DsxmCC\nBjz7EzQb8mC4zES+aDZnNDDDv9zAYGNzgMMSklykjXSEIpIYxxC0iFvn7uuBdwgKwDHA0+5e7+7r\nCJoOaVAAbIxl5WGhqTazXnHOLdJmKigiidFU0+B7Gw9QQdCuFwRtVY0zs739H00HKtuQTSQhVFBE\nEuNd4EIzSzazPIJuWj8G3ifoHCrJzAbw5ebD5wHDADzoK6MEuD1s9RgzG25m54bD/YCN3qibApGo\nqaCIJMbzwCyCFmanAjeGp7ieJWgRdg5B0/AfEfTQCEHfLCc0Wsckgk7EFoedkt3HF31wnAi8nNgf\nQaR11NqwSDszsyx3LwuPMj4GJrr7OjPrQXBNZeJeLsY3rOM54BZ3X9AOkUVioru8RNrfP8ysD0EX\nwz8Pj1xw9wozu42gX++VzS0cdvD0dxUT6Wh0hCIiInGhaygiIhIXKigiIhIXKigiIhIXKigiIhIX\nKigiIhIXKigiIhIX/w/Aq6y8Edb7AAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x301778d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# dict with classes as the keys, and the values as the grid of scores obtained during cross-validating each fold,\n",
    "# Each dict value has shape (n_folds, len(Cs))\n",
    "n_Cs = len(Cs)\n",
    "n_classes = 3\n",
    "scores =  np.zeros((n_classes,n_Cs))\n",
    "print(np.mean(lrcv_L1.scores_[1],axis = 0))\n",
    "for j in range(n_classes):\n",
    "    scores[j][:] = np.mean(lrcv_L1.scores_[j],axis = 0)\n",
    "    \n",
    "mse_mean = np.mean(scores, axis = 0)\n",
    "plt.plot(np.log10(Cs), mse_mean.reshape(n_Cs,1)) \n",
    "#plt.plot(np.log10(reg.Cs)*np.ones(3), [0.28, 0.29, 0.30])\n",
    "plt.xlabel('log(C)')\n",
    "plt.ylabel('neg-logloss')\n",
    "plt.show()\n",
    "\n",
    "#print ('C is:',lr_cv.C_)  #对多类分类问题，每个类别的分类器有一个C"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.00000000e+00,  5.00800483e-01,  0.00000000e+00,\n",
       "         0.00000000e+00, -8.88914767e-04, -9.57251506e-02,\n",
       "        -4.68696169e-02,  5.85329186e-03,  1.30309739e-04,\n",
       "         0.00000000e+00, -2.95246453e-02,  3.90704268e-02,\n",
       "         3.56097184e-08, -1.52677496e-03, -1.09208569e-03,\n",
       "         1.35824354e-03, -8.79745845e-04,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00, -5.86668846e-03, -1.15388667e-01,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  1.43760406e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         8.80862065e-01,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00],\n",
       "       [ 0.00000000e+00,  1.27547121e-01, -4.04428521e-03,\n",
       "         2.05869790e-03, -3.46148002e-04,  4.52329471e-02,\n",
       "         4.55255688e-03,  4.41288748e-03, -2.69013691e-04,\n",
       "         0.00000000e+00,  1.51327576e-03,  6.89996924e-02,\n",
       "        -2.36265288e-08, -9.08161415e-04, -1.78808055e-04,\n",
       "         6.50907225e-04,  2.31228601e-05,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00],\n",
       "       [-2.72760142e-01, -6.32806007e-01,  0.00000000e+00,\n",
       "         0.00000000e+00,  9.35994342e-04, -3.06106663e-02,\n",
       "         8.60930598e-03, -6.28730254e-03, -1.59648175e-03,\n",
       "        -1.90817655e-04,  1.05935517e-02, -1.05773884e-01,\n",
       "         4.03308265e-07,  1.23981961e-03,  7.12310267e-04,\n",
       "        -9.37424743e-04,  3.08883484e-04,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  4.76027537e-01,\n",
       "        -1.42779145e-01,  1.05392618e-01,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  2.50359591e-01,\n",
       "         0.00000000e+00,  0.00000000e+00, -1.20008851e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "        -1.14708437e-01,  0.00000000e+00, -7.65706327e-01,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         9.82553174e-02,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00, -3.40600928e-01,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "        -3.38935759e-01,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  1.19898377e-02,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  1.63533167e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "        -2.30095248e-01,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         1.49335846e-02,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00]])"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lrcv_L1.coef_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best C  is: [0.5 0.1 0.6]\n"
     ]
    }
   ],
   "source": [
    "print ('best C  is:',lrcv_L1.C_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "LogisticRegressionCV + L1正则 最优C选择"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classification report for classifier LogisticRegressionCV(Cs=[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7, 1.8, 1.9, 2.0, 2.1, 2.2, 2.3, 2.4, 2.5, 2.6, 2.7, 2.8, 2.9, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6, 3.7, 3.8, 3.9, 4.0, 4.1, 4.2, 4.3, 4.4, 4.5, 4.6, 4.7, 4.8, 4.9, 5.0, 5.1, 5.2, 5.3, 5.4, 5.5, 5.6, 5.7, 5.8, 5.9, 6.0, 6.1, 6.2, 6.3, 6.4, 6.5, 6.6, 6.7, 6.8, 6.9, 7.0, 7.1, 7.2, 7.3, 7.4, 7.5, 7.6, 7.7, 7.8, 7.9, 8.0, 8.1, 8.2, 8.3, 8.4, 8.5, 8.6, 8.7, 8.8, 8.9, 9.0, 9.1, 9.2, 9.3, 9.4, 9.5, 9.6, 9.7, 9.8, 9.9],\n",
      "           class_weight=None, cv=5, dual=False, fit_intercept=True,\n",
      "           intercept_scaling=1.0, max_iter=100, multi_class='ovr',\n",
      "           n_jobs=1, penalty='l1', random_state=None, refit=True,\n",
      "           scoring='neg_log_loss', solver='liblinear', tol=0.0001,\n",
      "           verbose=0):\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.00      0.00      0.00         7\n",
      "          1       0.50      0.14      0.22        28\n",
      "          2       0.67      0.95      0.79        65\n",
      "\n",
      "avg / total       0.58      0.66      0.58       100\n",
      "\n",
      "\n",
      "Confusion matrix:\n",
      "[[ 0  1  6]\n",
      " [ 0  4 24]\n",
      " [ 0  3 62]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\grid_search.py:42: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. This module will be removed in 0.20.\n",
      "  DeprecationWarning)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\learning_curve.py:22: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the functions are moved. This module will be removed in 0.20\n",
      "  DeprecationWarning)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    }
   ],
   "source": [
    "#使用测试集对模型进行预测\n",
    "from sklearn import *\n",
    "pre_test_logis_L1=lrcv_L1.predict(X_test)\n",
    "print(\"Classification report for classifier %s:\\n%s\\n\"\n",
    "      % (lrcv_L1, metrics.classification_report(y_test, pre_test_logis_L1)))\n",
    "print(\"Confusion matrix:\\n%s\" % metrics.confusion_matrix(y_test, pre_test_logis_L1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#对测试集进行预测\n",
    "def test_data_pred(lr,X_test):\n",
    "    pre_test=lr.predict(X_test)\n",
    "    out_df = pd.DataFrame(pre_test)\n",
    "    out_df.columns =['result']\n",
    "    out_df.replace({0:\"high\",1:\"medium\",2:\"low\"},inplace=True)\n",
    "    #out_df[\"listing_id\"] = X_test.listing_id.values\n",
    "    out_df.to_csv(\"test_result.csv\", index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 预测"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_data_pred(lrcv_L1,test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### L2正则"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegressionCV(Cs=[0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1, 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2, 0.21, 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.3, 0.31, 0.32, 0.33, 0.34, 0.35, 0.36, 0.37, 0.38, 0.39, 0.4, 0.41, 0.42, 0.43, 0.44, 0.45, 0.46, 0.47, 0.48, 0.49, 0.5, 0...85, 99.86, 99.87, 99.88, 99.89, 99.9, 99.91, 99.92, 99.93, 99.94, 99.95, 99.96, 99.97, 99.98, 99.99],\n",
       "           class_weight=None, cv=5, dual=False, fit_intercept=True,\n",
       "           intercept_scaling=1.0, max_iter=100, multi_class='ovr',\n",
       "           n_jobs=1, penalty='l2', random_state=None, refit=True,\n",
       "           scoring='neg_log_loss', solver='liblinear', tol=0.0001,\n",
       "           verbose=0)"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegressionCV\n",
    "\n",
    "#Cs = [1, 10,100,1000]\n",
    "Cs=[i/100 for i in range(1,10000)]\n",
    "\n",
    "# 大量样本（6W+）、高维度（93），L2正则 --> 缺省用lbfgs，为了和GridSeachCV比较，也用liblinear\n",
    "\n",
    "lr_cv_L2 = LogisticRegressionCV(Cs=Cs, cv = 5, scoring='neg_log_loss', penalty='l2', solver='liblinear', multi_class='ovr')\n",
    "lr_cv_L2.fit(X_train, y_train)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: array([[-0.27721437, -0.27721437, -0.27721436, ..., -0.27721436,\n",
       "         -0.27721436, -0.27721436],\n",
       "        [-0.26740198, -0.26740196, -0.26740196, ..., -0.26740195,\n",
       "         -0.26740195, -0.26740195],\n",
       "        [-0.3155916 , -0.31559168, -0.3155917 , ..., -0.31559176,\n",
       "         -0.31559176, -0.31559176],\n",
       "        [-0.24918631, -0.2491863 , -0.2491863 , ..., -0.2491863 ,\n",
       "         -0.2491863 , -0.2491863 ],\n",
       "        [-0.27272327, -0.27272327, -0.27272327, ..., -0.27272327,\n",
       "         -0.27272327, -0.27272327]]),\n",
       " 1: array([[-0.56079332, -0.56079333, -0.56079333, ..., -0.56079333,\n",
       "         -0.56079333, -0.56079333],\n",
       "        [-0.54098011, -0.54098011, -0.54098011, ..., -0.54098011,\n",
       "         -0.54098011, -0.54098011],\n",
       "        [-0.52214109, -0.52214108, -0.52214108, ..., -0.52214108,\n",
       "         -0.52214108, -0.52214108],\n",
       "        [-0.55450711, -0.55450713, -0.55450713, ..., -0.55450715,\n",
       "         -0.55450715, -0.55450715],\n",
       "        [-0.52778551, -0.5277855 , -0.5277855 , ..., -0.5277855 ,\n",
       "         -0.5277855 , -0.5277855 ]]),\n",
       " 2: array([[-0.61479676, -0.61479677, -0.61479678, ..., -0.61479678,\n",
       "         -0.61479678, -0.61479678],\n",
       "        [-0.57645916, -0.57645913, -0.57645912, ..., -0.5764591 ,\n",
       "         -0.5764591 , -0.5764591 ],\n",
       "        [-0.58033394, -0.58033396, -0.58033396, ..., -0.58033393,\n",
       "         -0.58033395, -0.58033393],\n",
       "        [-0.60207948, -0.60207951, -0.60207951, ..., -0.60207953,\n",
       "         -0.60207953, -0.60207954],\n",
       "        [-0.59289791, -0.59289792, -0.59289792, ..., -0.59289793,\n",
       "         -0.59289793, -0.59289793]])}"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr_cv_L2.scores_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[-0.54124143 -0.54124143 -0.54124143 ... -0.54124143 -0.54124143\n",
      " -0.54124143]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY0AAAERCAYAAACHA/vpAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3XeYVOXZ+PHvvZ3dZWEbvcOiggLq\nCoqCsXcwsSXRqEmIaUb95U3R10STGBOjxiRGk1iSGBNN87WjUWwBCyggVRSkSIcFFhZ22X7//jhn\nhtnZM7NndmZ2ttyf65qLKc+c88wA556n3Y+oKsYYY4wfaamugDHGmK7DgoYxxhjfLGgYY4zxzYKG\nMcYY3yxoGGOM8c2ChjHGGN+6bdAQkT+JyE4RWZGg490pIitFZJWI3CsiEuP7LxYRFZFyj9cOE5El\nIbcqEbnBfa1IROaIyBr3z0L3+Zkisswtv1BETnKfnyQi77h1XSYil4WcR0TkdhFZ7X6O69znD3ff\nUyci34nnewo513QRWSwijSJycSKOaYxJvW4bNIBHgLMTcSARmQqcCEwAjgSOA06O4f29geuABV6v\nq+pHqjpJVScBxwI1wFPuyzcCr6pqGfCq+xj3/kT3PV8CHnafrwGuVNXxOJ//1yLS133tamAocLiq\nHgH8w31+j1u/u/1+Jh82uud7PIHHNMakWLcNGqo6F+diGCQio0XkPyKySETmicjhfg8H5ABZQDaQ\nCeyIoTq3AXcCtT7KngasVdVP3Mczgb+49/8CXAigqgf00MrMPLeOqOpqVV3j3t8K7ARK3XJfB36i\nqs3u6zsDf6rqe0BDeGVE5AoReddt0TwgIul+PrCqblDVZUCzn/LGmK6h2waNCB4EvqWqxwLfAX7n\n502q+g7wOrDNvb2kqqv8vFdEjgaGqurzPuv4WeDvIY/7q+o2tx7bgH4hx/60iHwIzMZpbYSfezJO\noFvrPjUauMztznpRRMraqPsRwGXAiW6Lpgm43OfnMMZ0QxmprkBHEZF8YCrw75DhiGz3tc8AP/F4\n2xZVPUtExgBHAEPc5+eIyHRVnSsijwLHeLz3d8AfgF/hdNP4qWMWMAO4yU95VX0KeEpEpuO0Zk4P\nOdZA4K/AVYGWBc7nrVXVcvcz/wmYFuUUp+F0l73nfme9cFouRPvcquorGBtjup4eEzRwWlV73V/M\nLajqk8CTUd77aWC+qh4AEJEXgeOBuap6ZaQ3iUgfnDGQN9yL7gDgWRGZoaoLPd5yDrBYVUO7vnaI\nyEBV3eYGgp0e9Z/rdr2VqOouESnAaX38QFXnhxTdDPyfe/8p4M9RPjOAAH9R1VZBLNrnNsZ0Xz2m\ne0pVq4D1InIJBGcSTfT59o3AySKSISKZOIPgbXZPqeo+VS1R1RGqOgKYD0QKGACfo2XXFMCzwFXu\n/auAZ9z6jwnM4BKRY3C6oXa7rZWngEdV9d9hx3oaONW9fzKwuo2P8CpwsYj0c89TJCLD23iPMaY7\nU9VuecO5+G7DGdzdDHwZGAn8B1gKfADc4vNY6cADOIHiA+CedtbpDaDcvT8IeCHktVxgN9An7D3F\nOBfvNe6fRe7z3wdWAkuAd4CT3OevcD/zkpDbJPe1vjgtkOXueya6zw9wv6MqYK97v8B97TL3GMuA\nRcDxPj/rce5xqt3PtTLV/ybsZje7xX8TVUuNbowxxp8e0z1ljDEmfikZCBeRu4ALgHqc6aBfVNW9\nHuX+HzALZw3Ccrdc1LUOJSUlOmLEiITX2RhjurNFixbtUtXStsqlpHtKRM4EXlPVRhH5BYCqfj+s\nzGDgTWCcqh4UkX/hjAE8Eu3Y5eXlunBhpHFmY4wxXkRkkaq2SnMULiXdU6r6sqo2ug/nc2j9Q7gM\noJeIZOAMFG/tiPoZY4zx1hnGNL4EvBj+pKpuwcmFtBFnFtQ+VX3Z6wAico27ynlhRUVFUitrjDE9\nWdKChoi8IiIrPG4zQ8rcDDQCj3m8vxAn79JInOmpeSJyhde5VPVBVS1X1fLS0ja75IwxxrRT0gbC\nVfX0aK+LyFXA+cBp6j2wcjqwXlUr3PJP4qQB+Vui62qMMcaflHRPicjZOIvTZqhqTYRiG4HjRSTX\nXfl8Gj5WYRtjjEmeVI1p3Af0xkn8t0RE/gAgIoNE5AUAVV0APAEsxplum4aTpdYYY0yKpGSdhqqO\nifD8VuDckMe3Ard2VL2MMcZE1xlmT3UKe2vq+c0ra1ixZV+qq2KMMZ1WT0qNHlVamnDva2toaGrm\nyMF9Ul0dY4zplKyl4SrIyeTooX2Zt8bWeRhjTCQWNEJMKytl2ZZ9VFbXp7oqxhjTKVnQCDFtbAmq\n8NbaXamuijHGdEoWNEJMGNyHgpwM5q62LipjjPFiQSNERnoaJ5WVMG/NLmxzKmOMac2CRphpZaVs\n21fL2ooDqa6KMcZ0OhY0wpw0pgSAuattXMMYY8JZ0AgztCiXUSV5zLWpt8YY04oFDQ/Tx5Yyf91u\n6hqbUl0VY4zpVCxoeJhWVkJtQzOLNlSmuirGGNOpWNDwcPyoYjLThblrbFzDGGNCWdDwkJedwTHD\nCm29hjHGhLGgEcH0saV8sK2Kiv11qa6KMcZ0GhY0Iphe5uw1/tbH1kVljDEBFjQiGD+ogMLcTOui\nMsaYEBY0IkhLE04qK2WupRQxxpggCxpRTC8rYdeBOj7cvj/VVTHGmE7BgkYU09xxDduYyRhjHCkJ\nGiJym4gsE5ElIvKyiAyKUO4qEVnj3q7q6HoO6JPD2P75lofKGGNcqWpp3KWqE1R1EvA8cEt4AREp\nAm4FpgCTgVtFpLBjq+m0Nt7dsIeD9ZZSxBhjUhI0VLUq5GEe4DXSfBYwR1X3qGolMAc4uyPqF2r6\n2FLqG5t5d8Oejj61McZ0Oikb0xCR20VkE3A5Hi0NYDCwKeTxZve5DjV5RBFZGWnMs6m3xhiTvKAh\nIq+IyAqP20wAVb1ZVYcCjwHXeh3C4znPua8ico2ILBSRhRUVib2498pKZ/KIIkuVbowxJDFoqOrp\nqnqkx+2ZsKKPAxd5HGIzMDTk8RBga4RzPaiq5apaXlpampgPEGJaWQmrdxxg+77ahB/bGGO6klTN\nnioLeTgD+NCj2EvAmSJS6A6An+k+1+Gmj7Wpt8YYA6kb07jD7apahhMMrgcQkXIReRhAVfcAtwHv\nubefuM91uMMH9KYkP5t5lirdGNPDZaTipKrq1R2Fqi4EZoU8/hPwp46qVyQiwvSyEt5YXUFzs5KW\n5jXcYowx3Z+tCPdp+thS9lTXs3JrVduFjTGmm7Kg4dOJY0oAbBaVMaZHs6DhU2nvbMYNLLBU6caY\nHs2CRgymjS1h8cZKDtQ1proqxhiTEhY0YnByWSkNTcqCdbtTXRVjjEkJCxoxOHZEITmZaTb11hjT\nY1nQiEF2RjrHjyq2cQ1jTI9lQSNG08pKWbermk17alJdFWOM6XAWNGJ08lhn6u2bH1sXlTGm57Gg\nEaPRpfkM7JNjeaiMMT2SBY0YiQjTykp4c80uGpuaU10dY4zpUBY02mFaWSlVtY0s27Iv1VUxxpgO\nZUGjHU4aU4IIzFtt4xrGmJ7FgkY7FOZlMWFwHxvXMMb0OBY02mlaWSnvb9pLVW1DqqtijDEdxoJG\nO00rK6GpWXn7Y0spYozpOSxotNMxwwvJy0q3LipjTI9iQaOdMtPTOGF0CXPXVKCqqa6OMcZ0CAsa\ncZg+toRNew7yyW5LKWKM6RksaMRhWlkpgHVRGWN6DAsacRhRnMvQol7MtVTpxpgeIiVBQ0RuE5Fl\nIrJERF4WkUEeZSaJyDsistIte1kq6hqNk1KklHfW7qbBUooYY3qAVLU07lLVCao6CXgeuMWjTA1w\npaqOB84Gfi0ifTuykn5MLyvhQF0j72/cm+qqGGNM0qUkaKhqVcjDPKDV9CNVXa2qa9z7W4GdQGnH\n1NC/E0aXkJ4mNq5hjOkRUjamISK3i8gm4HK8WxqhZScDWcDajqhbLPr0ymTS0L42rmGM6RGSFjRE\n5BURWeFxmwmgqjer6lDgMeDaKMcZCPwV+KKqeg4ciMg1IrJQRBZWVHT8L/5pZSUs27yXyur6Dj+3\nMcZ0pKQFDVU9XVWP9Lg9E1b0ceAir2OISAEwG/iBqs6Pcq4HVbVcVctLSzu+B2taWSmq8NZaa20Y\nY7q3VM2eKgt5OAP40KNMFvAU8Kiq/ruj6tYeE4f0oSAnw1KlG2O6vVSNadzhdlUtA84ErgcQkXIR\nedgtcykwHbjanZq7REQmpai+UWWkp3HimBLmWUoRY0w3l5GKk6qqZ3eUqi4EZrn3/wb8rSPrFY9p\nZaW8uGI7aysOMKZf71RXxxhjksJWhCfItLISAOZaF5UxphuzoJEgQ4tyGVWSZ+s1jDHdmgWNBJpW\nVsL8dXuoa2xKdVWMMSYpLGgk0LSyUg42NLFoQ2Wqq2KMMUlhQSOBjh9dTEaa2OpwY0y3ZUEjgfKz\nMzh2eKGNaxhjui0LGgk2fWwpK7dWUbG/LtVVMcaYhLOgkWCBqbdvfWxdVMaY7seCRoKNH9SHwtxM\n5loXlTGmG7KgkWDpacJJZaXMW7PLUooYY7odCxpJMK2shIr9dXy4fX+qq2KMMQllQSMJAuMaNovK\nGNPdWNBIgoF9elHWL595tl7DGNPNWNBIkuljS1mwfg+1DZZSxBjTfVjQSJJpZSXUNzazYP2eVFfF\nGGMSxlfQEJETRSTPvX+FiNwjIsOTW7WubcrIYrLS05i32sY1jDHdh9+Wxu+BGhGZCHwP+AR4NGm1\n6gZ6ZaVz3MhCG9cwxnQrfoNGozqLDmYCv1HV3wC2PV0bppeV8tGO/WzfV5vqqhhjTEL4DRr7ReQm\n4ApgtoikA5nJq1b3MK2sFLCpt8aY7sNv0LgMqAO+rKrbgcHAXUmrVTdx+IDelORnWxeVMabbyPBZ\nbj9Ot1STiIwFDgf+nrxqdQ9pacL0shLeWF1Bc7OSliaprpIxxsTFb0tjLpAtIoOBV4EvAo+096Qi\ncpuILBORJSLysogMilK2QES2iMh97T1fKk0bW8Ke6npWbq1KdVWMMSZufoOGqGoN8Bngt6r6aWB8\nHOe9S1UnqOok4HnglihlbwP+G8e5UurEMU5KEct6a4zpDnwHDRE5AbgcmO0+l97ek6pq6M/uPMAz\nHayIHAv0B15u77lSrV/vHI4YWGCD4caYbsFv0LgBuAl4SlVXisgo4PV4Tiwit4vIJpxA1KqlISJp\nwC+B7/o41jUislBEFlZUdL6L8/SxJSz6pJLqusZUV8UYY+LiK2io6n9VdQbwOxHJV9V1qnpdtPeI\nyCsissLjNtM95s2qOhR4DLjW4xDfAF5Q1U0+6vegqparanlpaamfj9ShppeV0tCkvLJqR6qrYowx\ncfE1e0pEjsJZAV7kPJQK4EpVXRnpPap6us86PI7T5XVr2PMnANNE5BtAPpAlIgdU9Uafx+00pows\n4vABvbnrpY84a/wAcjLb3bNnjDEp5bd76gHg26o6XFWHAf8DPNTek4pIWcjDGcCH4WVU9XJVHaaq\nI4DvAI92xYABkJGexi3nj2Nz5UEenrcu1dUxxnQx2/fV8tLK7amuBuA/aOSpanAMQ1XfwBnAbq87\n3K6qZcCZwPUAIlIuIg/HcdxOa+qYEs4eP4D7X1/Ltn0HU10dY0wXctHv3+arf12U6moA/oPGOhH5\noYiMcG8/ANa396SqepGqHulOu71AVbe4zy9U1Vke5R9RVa9xjy7l5vOOoEmVX7zYqmFljDERbdnb\neX5o+g0aXwJKgSeBp9z7X0xWpbqroUW5XDNtFE8v2cqiT2yfDWNM1+N39lSlql6nqseo6tGqer2q\nVia7ct3R1z81mv4F2fzo2Q9obvZcnmKMMZ1W1NlTIvIcERbeAbjTcE0M8rIzuOmcI7jhn0t4YvFm\nLi0fmuoqGWMSRFV5dunWbj1Lsq0pt3d3SC16mJmTBvHoOxu48z8fcc6RA+idY1nmjekO5q7ZxfX/\nWMKXTxrJD88fl+rqJEXUoKGqXTbnU2cmIvxoxnhm3PcW9732MTede0Sqq2SMSYB9BxsA2FHVfTde\n87tH+HI3K23obZ6I/EpEipNdye5owpC+XHLsEP701nrWVRxIdXWMMQngbHB6yOsf7uTF5dsilv9g\na1Uw0LTH+l3VbNpT0+73t4ff2VMv4qzavty9PQfMA7YTR4r0nu67Zx9GdkY6t89eleqqGNPp7Nxf\nS0NTc6qr0S4izt45X3zkPb7+2OKI5c69dx6fe3B+u89zyt1vMO3OuNIAxsxv0DhRVW9S1eXu7Wbg\nZFX9BTAiedXr3vr1zuFbp47h1Q938sZHO1NdHWM6jZr6Ribf/io/fHpFqquSdB9s61p77fgNGvki\nMiXwQEQm4+SDArDUrXG4+sQRjCjO5SfPf9Blf1UZk2gH65sAePmD9iX5fOWDHbz2YedMEHrvq2sY\ncePstgt2Un6DxizgYRFZLyIbgIeBWSKSB/w8WZXrCbIz0vnh+eNYV1HNo+98kurqGNMtzHp0IV96\nZGFSjv3U+5uZv2531DLRNna+Z87qxFaog/nKcquq7wFHiUgfnF389oa8/K+k1KwHOfXwfkwfW8qv\nX1nNhZMGUZyfneoqGdMphA8sdwb/759LAdhwx3kprklq+J091UdE7sHZH/wVEfmlG0BMAogIt5x/\nBAfrm7j75a79K8SYRNhf2zV7vTthjEs4v91TfwL2A5e6tyrgz8mqVE80pl9vrjxhBP94byMrtuxL\ndXWMSZndB+r41N1vpLoa7aJuAg2J1j8Vz/E7QVTyGzRGq+qt7o5961T1x8CoZFasJ7r+9DIKc7P4\nyXMfdIp/HMakQsWBuuD9rvq/IEkxo1PwGzQOishJgQciciLQeXL1dhN9emXynTMP490Ne3h+WeQF\nQcaYzqkn/NbzGzS+DtwvIhtE5BPgPuBryatWz3XZcUMZN7CAn7+wKjjt0BiTOJXV9Yy4cTb/XV2R\nsGOqKm+u2RUMGpKk/qmRN73A/a9/nJRj++U3NfoSVZ0ITACOctOjL01u1Xqm9DTh1gvGsXVfLX/4\n79pUV8eYbmfFVmfM8KG5idt6+W8LNnLFHxfw/LKtCTtmJHe99FFKc1u1lRr92xGeB0BV70lCnXq8\nKaOKOW/CQP7w37VcetxQBvftleoqGRM0f91u+vTK5IiBBUk5fkd18ajPEZPGpmYUyEyP/Bs7kP9p\n617nYp7sMY2L//A28753aovn1u+qZmCfnKSnZG+rpdG7jZtJkv91M99e+ccFPDR3Xbv2FV9XcYBf\nvvwR722wXQIToa6xie8/sYyd3TiDqR+ffXA+5/xmXqqrETNV5an3NwczL/gNTpN/9ioTfvQyAFW1\n3skFq+ucKcIf7dgPwP665E4Z3rSn5fWguq6RU+5+g+8+sSyp54W2U6P/OOk1MJ4G9+3FvZ87mvtf\n/5jbX1jFz15cxXEjipgxcRDnHjWQorwsz/c1NDUz54Md/G3+J7y91lm1unhjJY/NOr4jq98t/WfF\ndv65cBM1DU389nNHp7o6JkYn/Pw1tlfVMi7GFtKe6vrg/UDwCLdgfcsfZnMipD9JVvqQ2gZn/PO5\npVuT/m/T14rwUCKyWFWPSUZlTEtnjR/AWeMHsH5XNc8t3cqzS7fyg6dX8KNnVzKtrIQZkwZxxrgB\n5GdnsGXvQf7x7kb+8d4mKvbXMbhvL7571mF8sruap9/fSk19I7lZMf91mxCBX6Zp3Xk+ZSdQEzIB\nJJFdVdvdFuKWvQcTfuz2aPLY7nnemgoG9+3FqFIntd/yzfu44L432zzWDf9ckvD6RdKeq0jc/2VE\n5DZgJtAM7ASuVtVWI0giMgwnz9VQnCnb56rqhnjP39WMLMnjutPK+NapY1i1bT/PLt3Kc0u38v/+\nuZSczOWMG1jAkk17UeDUw/px+fHDOHlsP9LThHlrKvjXws0sWL+HUw7rl+qP0qU1u1eZtGSt3DJU\n7K/jot+/HfP7lm3ey+odB7j42CFtlg3sX9HY7J0gdPWO/bzx0U6umT66xfOLPqmMuV7R3PnSh62e\n+8If3wUOpSiZHWUvjlDz1uxKXMXa0J6gkYj21V2q+kMAEbkOuAXvKbyPArer6hwRyccJMj2WiDBu\nUAHjBhXwvbMOY/HGSp5dupWFGyr5+qdG87nJwxhSmNviPceNKCI7I415q3dZ0IhT4JehxYzk2b6v\nfeNFM+57C8AzaFTVNvD80tYXX69f+s6x3qS2oblV0IgWzNqzGHfu6sgX+o+27+ewAb075XhkzEFD\nVX8Q70lVNTSBfB4eCz9FZByQoapz3PfY9nYh0tKE8hFFlI8oilouJzOdySOLmLcmcXPSe6pASyPd\nokaHSURmhJueXM7sGBbL1jYc+m364fbU7HVx1q/nsuGO8zplZgi/CQv3i0hV2G2TiDwlIu1KJyIi\nt4vIJpydAG/xKDIW2CsiT4rI+yJyl4h4ziUTkWtEZKGILKyosItjuOllpazZeaBdM7DMIYEfpuk2\nqAHAqXe/wb8Xbkp1Ndq0OyQtSSxWbNnH2b/uuFli9Y1doyPF74rwe4DvAoOBIcB3gIeAf+AkM2xF\nRF4RkRUet5kAqnqzqg4FHgOu9ThEBjDNPddxOLmurvY6l6o+qKrlqlpeWlrq8yP1HNPGlgAd2+/Z\nHR3qnrKgAbBuV3XSp3hWJSDbbXt/rJ//27YHoP36x7sbo76+esd+Ptq+v9Xz0ar+7Q4c/A7lN2ic\nraoPqOp+Va1S1QdxBqX/CRR6vUFVT1fVIz1uz4QVfRy4yOMQm4H33QSJjcDTgM3aaofD+vemtHe2\nBY04Bbun/P6vSYH7X/+422USWOjRr3+grjHugenFG/fy9PtbWjwXaR1GvG58cjmNYTtzhnZ9nfmr\nuWzZWxPTMZ8Mq3tH8fvPv1lELhWRNPd2achrMcdxESkLeTgDaD2NAN4DCkUk0HQ4Ffgg1nMZ55fx\ntLIS3lxTEXHwz7Stubnzz56666WPuOPFD6ltaGLana91i7GsZZudtB9zV1cw4sbZ7D5Qx9f/toiL\nfv82B+JcRBc+VfX4n70a8zE+3rnfM09cTX3LuoX/zwtvAb28snNuTxvOb9C4HPgCzvTYHe79K0Sk\nF95dS225w+2qWgacCVwPICLlIvIwgKo24XRNvSoiy3Gm+j7UjnMZnHGNypoGVm61vTraqzm4TqPz\nBo2ADbur2bTnILc97+931j0vf8STizcnuVbxeWiekytqxdYqlrt7zjQkeBygJsYkoZv21HD6PXPZ\n6jHr63evt2zxNbfRT+bVcuiE4+C+t3tdB1wQ4eWYO/5U1as7ClVdiLMfeeDxHJwkiSZOJ5UdGteY\nMKRvimvTNR3qnur8QUNiXE5172tO5tTPHNP2Oof2amxqprKmgdLeh7YzXvRJJf16ZzO0yJkqHike\nNzQ1+7qANjcrt7+wqsVzfq67m/bURFzFHc2uKIPs9WHdUTurYhuQ37r3IEs37227YAfzFTREZCzw\ne6C/qh4pIhOAGar606TWziRMSX424wcVMHd1Bd88ZUyqq9MlNQW7p1JckST75uOLOXNcf2ZOGpzQ\n49767EoeW7CRlT8+i7xs59ITWPvw3LUnsWD9bs9WnAJlN7/o6xwrtu7jj2+uj6leI26cTVZ6WquL\nfFs27amJOikifDZUrGOKU+94LabyHcVv99RDwE1AA4CqLgM+m6xKmeSYVlbK4o2VcfcD91RdqXsq\nIPTXeXVdY3A1dDSzl23j+n8kfmbOf1ZsB7y7gBZ+soefzl7FT3x2p0US3hqp9vlvPdaAATDtztej\ntuceeXtDi8dtdU91FX6DRq6qvhv2nF15upjpZSU0NCkL1u1OdVW6pGAakU7W1Fi/q7rV/gqbK52Z\nOKGXqeN/9ioTf+ydcC/Z9lTXs9tN/JeImLu3xjv4hV+Wx9/6UvwniyKWz9IZF+q1h9+gsUtERuP+\nnYjIxYDtR9rFHDuikJzMNJt6G4MNu6qD3R2B2VOdbUX4KXe/wZSwWT9f/svCVuX8pOtuz26RoVNH\nD9Q1cuszK1od55jb5kQ9RnuvpwfqGltkjvW6ML+7vnOk4ugeIcN/0Pgm8ABwuIhsAW7AtnvtcrIz\n0jl+VDFzE7jNZawO1DWydW/XWZn++Yfmc9vzH3CgrpEm7RxjGhfe/xZffuQ9X2VVlcrqeu59dY2v\n8p9pR7LAyx9awLqKA6zZsZ+v/20Rf3nnEx59Z0PE8oH1FftD1kREu6C+szZyy7iypj7iax0hlgkH\nzd1kurvfoLEF+DNwO84q8DnAVcmqlEmeaWWlrNtVHdxprKP99PkPuOzBd1Jy7mg2V9Yw4sbZvBCW\nVXS/uyK5sam506xxWbJpL69+uNNXWQW++Mh73DNnta/yq7a1zrW0ZNNelkWZxdOsyqm//C9n/Gpu\nsBXbrFD+0zn8z79a7wodCCiXPTD/UD2jNDVeWRV5VtMDYVu2dvTf0J4YgtbdL/v7O+js/AaNZ3Cm\n3DYAW4EDQHWyKmWSZ7o79fbNjzu+i0pVee3DnWzdW5vU/t1V26raTNsQ7oOtzsUyfK1CYPyivrGZ\nhianztFix46q2pTlY/IKao1NypJN7Zu2+cnuajZX1nDh/W8Fs8h68fo6Ptxexa4D9fxflLUfgV3u\n4hFLIsJkuOpP4UO9kXWXCSh+s9wOUdWzk1oT0yHG9MtnQEEO89ZU8LnJw+I61sc7DzCwT05w+mRb\nVu84wM79zlz1qtpG+vTKjOv8kfzzvU38a+EmPtuuz9eyuyHQFVXX2BxMA9EUJeBd/ef3WLWtitOO\n6M+KLfsQcVp3ydDUrC3WjHgF4nhaRyff9UaLxw/PW8esaa3zk3p1u7zozpTysnBDJQ/PW9fuunWu\nEaWex29L420ROSqpNTEdQkSYPraEN9fsiuuCcqCukfPuncef3/I/Jz40pcXeJPZFV9c1BveBjl3L\n7yQwvbausYnG5kBLI/L3Fljs1dDUzJV/eje4qU4yXP7w/BaPk90189PZqzxzM3mdN9qFva6xmZ/O\nXhWlRHRXxvDr3iSe36BxErBIRD4SkWUistxNAWK6oGllpVTVNkbtp27Lwg17qGtsZnNl5EHt5mZl\nxZZDaUtCZ21VRpgymQg19U3ZxFQYAAAgAElEQVQ0NGlMXWCRFmkFuqdqG5qDc/mjDWhmuOUbEzj+\n8cySLfziP63Ts81f13JWkNfHTfTagOl3vt7qOa9TxDrBLFHV/PfCzp0KpTvwGzTOAcpw8kRdAJxP\n5LQippM7cUwJIvDMkq3tHltY4E5j3HUgcovhyfe3cP5v3+Rf722itqGJBet3M3FIHyDyrJe7X/qI\n378RX5bWajdRnNeFe96aCiqr/bdyPLunmp19pr3yeAW6i5qa2v5eN1fW8JnfvdVmfa7/xxJf34l6\n/Ob3+g72h7UWvAa/I/FaH+EVmEJnFXXkBIK/xziWZWLnK2io6idet2RXziRHUV4W5x41kEfe3sBX\nHl3Izv2xb7E5310guLs6cj6dJxY5A8I/em4lTyzaTG1DMxdMHAQc6p46WN/UojXyzNItzPkgcn+4\nHzV1zhqB8C6qusYmvvDHd7nijwsivjf8+hfonqpvbKax6VD31Il3vMZ597ZMu7Zxd03wF3ak/adD\n/f6NtSzeuJejb5vDSyvj+8xw6HOH8moVHfWjl/n5i6uCaS7O+U18Gw15rfAObWmM/t8X2jxGeL4o\n03l14p0BTDL99rNH88PzxzFvzS7O/NVcnlu61fd7q+sag+mqd0doaWyurGH+uj1cPmUYWRlp/PCZ\nFWSmC+ccNRCAymrnF+vj727kwvvfYl9NA03Nyra9teyJoSXgpabBaWk0hP3aDyw4W7m19S/rSC2u\n0DGNhihjGpv21DD9rtfZtOegZ5kdVbWtUlocbDh0sf3Xe61nXDU2NcfUEjzaYwFdpEH7B/67jn91\ngV33TOdjQaOHSksTvnzSSGZfN43hxXl86+/v883HF/u6YC/6pJKmZmVs//yIW2k+s8QJQl87eTS/\nuGgCqnD0sEIGFuSQJoe6pzZX1tDYrKzbdYAdVbU0Nmv8QSNCSyNa2utIYxAS0j0VSMPt1d2yO6zO\noQFr1bYqpvzs1WByvoDahsj12VfTwJibX2y1DiFW0bqG6pK4vWhXys9lYuN3yq3ppsb0y+f/vnYC\nD8xdx69fWc32fbU88bUTombvnL9uN+lpwpnjBnDf6x9zsL6JXlmHtm9XVZ5cvJnJI4sYWpTL0KJc\nfv6Zoxhdmk9amtCnV2YwaFS4U3A37K4OXrirap3ZT5nt3CIvOKYR1tKIFjQCASb8EnuopdEcLOPV\n0ghPLRKaCyrQ/fOhu53ntn0H+e1rHwcXDnoJdBnGu+Yj2jmSuULZQkb3ZUHDkJGexjdPGUPf3Exu\nfmoFb6yu4JTD+kUsv2D9HiYM6cMwdw+EXQfqgvshACzfso+1FdV8JWROf+iakMLcrODsqUDQWL+r\n5Qr1ypp6+vXOadfnidTSiJZXKTyNdUBgYLuuoSk43dRr3+rwPTau/nPkNB83P7WC19pY0R24nIf+\nYn9zza7gviiJEG29SbxqG2PPYWW6BuueMkGXHDuUIYW9+NWc1RH70mvqG1m6aS9TRhZTnJ8FtO6a\neXLxFrIy0oLjF+EK87KCA+EVbvfWhl3VbAmZvhsY84iVqgZbGq27pw5d7OvCLmp+uqcC3WaBQBc4\nXyyam9XXbKJAaya0AXPFHxfwrb+/H9P5oknmrKbw8STTfVjQMEFZGWlcd2oZyzbv49VV3r+EF3+y\nl8Zm5fhRRRTnOzuwhY9rPL9sG2cc0T/iiu/C3MxgUNgV0j0VuuYj2qysaOoam4NpPsIDQU3IGEL4\nLmrB7qmwIBDaPRVoHdWFHKe+KfI4h5dZjy70tYYhUI3whHixTFhoS3dJoGc6lgUN08JnjhnM8OJc\n7pmz2vOiEhjPKB9RREmgpREyg2p/bQO7DtQxcWifiOfom+u0NGobmoJdPet3OUEj1x0baW9LI3Tc\nIrzLKbR7Knz/iYjdU+4V/snFm4MtjdAB5MD7GnxMsQV47cOdnv394YHEq6WRaIlcgGh6DgsapoWM\n9DSuP62MD7ZV8bLHeokF63dz5OA+5GdnUJzntDR2hbQKAhfj/gWRxyMKczOprGkIptw4rH9v9tc2\nsmLrPo4c5ASbPe1saYROa23V0mgRNFoev63uqdBpul5BI96unkDLYtu+g/zy5Y9arQFJhmbVlKbJ\nN12TBQ3TyoyJgxhVmsev5qxp0do4WN/Ekk17OX5kEQC9stLJy0pv0dLYts8JGgOiBI2+uVkcbGgK\nrmk4bmQh4Kw2Hj+4AIA9CWhptB4IPxRQwoNSYDpt4NNWVtdz3d/fbzVeAy2nyga6p8JnakUTbWba\n5Q8v4LevfeyrbLyamtXyOJmYpSRoiMhtbg6rJSLysogMilDuThFZKSKrROReSeb/IBMUaG18tGM/\nL6xwUk83Nyuvf7SThibl+FHFwbLF+dnBFgPA9kDQ6BOtpeF0a32805mCetyIouBrI0vyKMjJaPfm\nOtUhgSHaOo0F6/fw/LKtrcoGYuQjb2/g2aVbWwx6B3i1NKKtAB9VktficbSZU7vCztfWP/hrH1/c\nRonIkjl7ynRfqWpp3KWqE1R1EvA8cEt4ARGZCpwITACOBI4DTu7QWvZg508YRFm/fH42exWfe3A+\nE3/8Mt94bDHZGWkcO6IwWK44P6tFS8Nv9xQ4qdIBjhlWGJyyOrhvL4rysjx/4ftxsEVLw7t7qndO\nBs8v28a1jx+aiRRY7R3IL5WZ3vpy/dQ3pnL4gN4tB8KDQSPyBXhUab6vul/yh7dbTeeN9jNpSGEv\nno9jPwkbCDftkZJ1GqoamschD+/sygrkAFk4P7gygchbeJmESk8TbjzncK7/xxJqGpqYefQgJgzu\ny+SRRRTkHJoVVZKf3WIXwO1VtfTNzSQnM93rsIDTPQWw2t2Ep39BDkMLe7Fhdw1DCnMpzMuKKalg\nqBZjGuHdUw1N5GSmUZqf3WrRW6B7KtDiyM44VP/RpXk88sXJDC3KJT1NWrQ0AvejdU/5yTTbrMp7\nGypbPe+VIDBRHn3H0seZ2KVscZ+I3A5cCewDTgl/XVXfEZHXgW04QeM+VfXMaiYi1wDXAAwbFt/G\nQuaQ047oz4ofnxW1TEl+Voud4bbvq4s6ngFOwkRwNnHqm5tJVkYaI0vy2LC7hsGFvSjOy2LL3tiT\nKEL4mEZ4S6OR3KwMivKyWLer5caTgWBR774ndNzipDElwcWLaSItWhV1wYHwyN1TfgbJI7VUtkTZ\nTz3ewfdkphEx3VfSuqdE5BURWeFxmwmgqjer6lDgMeBaj/ePAY4AhgCDgVNFZLrXuVT1QVUtV9Xy\n0tLk7JJmvBXnZbOnuj7Y1bG96mDU8Qw41D21u7qeUnetx6ShhYwoziU/O8NZMR6lpbFt30E27vbe\n47ytMY1emenBoAWH1mUEuqcCLY7QzYZC05mkha38Dk65jdLS+K+PGUrtCQCdZc9y07MkraWhqqf7\nLPo4MBu4Nez5TwPzVfUAgIi8CBwPzE1YJU3civOzaGpW9h1soDAvi+376oLTZiMJdE8BlPZ2gsY3\nTxnNV0920o4U5Wexp7oeVfWcPXTLMyuprK7nia9PbfVaaHrw8MHpg/VN5Galk5mRFlJGyUyXYLD4\nYFsVd730IfsOHgoaGSFBI3yoI9bFfZG0Z81EojdYMsaPVM2eKgt5OANovS0ZbAROFpEMEcnEGQS3\npPudTHBVeHUdDU3N7K6uizoIDs7K8zx3EV+J+/6M9LTgOEhRbhb1Tc1UR8gVVbG/Lph+BJwFhdc8\nupD7XlvTsnuqsfVAeG5WeouFfIHWSGir5P7X11J18FCLJSskUoRnb61vbP3+9mhP0LHFeSYVUjV7\n6g63q2oZzm6A1wOISLmIPOyWeQJYCywHlgJLVfW5lNTWRFTidvVU7K9n5/46VKNPtw0ItDYCLY1Q\nhe4xI3VRVR1soCqkJTDrLwt5+YMd/HvR5hb5pcJXaQey8YYGjU9211Df2BzsngqeI6R7KrTVEd49\n9ZVHF7J9X23cLY21FQdifo+f3QGNSbRUzZ66KMLzC4FZ7v0m4KsdWS8Tu5Leh1oaWW63T1sD4QCF\neZls2XvQM2gU5x1KhBiaPTegqraBqtpGVJXahmYWrN9D39xMPtldw+bKg4g4K6wbwgZ6axoa6dc7\nhyuOHxYcZzjnN/OYOWlQq7Jvr90dvL+24tCgeXgKdICH561jeNhajFi1Z5aUtTRMKtiKcBOX4AX+\nQL2vhX0BgQV+gYHwFq9FaWmoKlUHG2lqVmrqm9jurgs544j+ALy0cjvjBjqryr3SiPTKSufUw/tz\n+6ePDD4/e9k2z+6l/gVO3Q4f0Dv4XHgKdIDMjDSa4uyeao+DUTZxMiZZbD8NE5e+uVmkiZPpNnCR\n9tPSiNY9FdrSCFfX2BwcfK6qbWDbPmdK6pnjB/DE4s00NiufOWYIK7d+ECwXcLC+iVx33CQzreVg\nuNev9j9edRy9czIY2KdX8DmvxXZZ6Wkx/+pPTxOb/WS6JGtpmLikpwlFeVnsqq5nR1UtWRlp9M31\nTokeKjDttsSjpdG/IIfMdGGNu/gvVOhYRtXBxmDrZky/fMr65ZORJlww0dnHI3zB3f7aRnq7CxMz\nM7wHtANuu/BIjhzch+HFecFut8DnDZeVEXvQyGrnroTGpJq1NEzcivOy2X2gjgO1jQwoyPGVZK8w\nSksjJzOdSUP78s663a1eCx2UdloahxIkfuGEEezYVxvs8gpdEd7crByoayQ/x/knn5HW8qId2j31\n1DemcvSwQrx4jWlkpkvMmw5lZ6ZZ95LpkixomLiV9M5i14F60tPEV9cUwNlHDmB/bWNwT45wJ4wq\n5r7XP6aqtqFF2pLQWU1VBxvYvs9JW9IrK50vHD88+FpWelpwdTccWvRX4AaN8NxSoRf94cWRB7UD\nATG0e+m5pdv4YFtVxPd4sZaG6arsX66JW6ClsX1fLf19DIIDHDGwgFsuGBexVXL86GKaFd5dtweA\n1z7cwSV/eLtFyvSq2ga2V9V6BqqMdGnR0gjkmsrP9m5pLN+yj+HFudx87hEtVoyHC1zrQ4PO8i37\nYh6fCO3yMqYrsX+5Jm7F+U5LY0dVLQN9Bo22HDOskKyMtGAX1TNLtvLehkrW7Dw0zhEY0/CarZWZ\nnkZDUzM19Y3srKrlgJvIMNg95ZHFtrK6nq9MHxW1XoExjcy0+P7rZMcZNDI8xlaM6QgWNEzcSvKz\nOVDXSF1jc5urwf3KyUznmGF9ecddL7HoEycD7KptoUHDGdPwClSZ6UJDs3LXSx9x0R/eZr/brRUY\nCPfqHrrz4glt1iuwIjwzzot+tNaMH+Er043pKBY0TNyKQy6Afsc0/DhxdAmrtlfx/sZKNlc6U2tX\nuWMHmenC7up6dh2oY0BBr1bvzUxPo6GxmWWb97Fpz8HgZkrB7qmwoHHxsUM4+8iBbdYpL8t5v1d3\nVPlw78FzL/16x/c9WcwwqWJBw8StOGTa7IA+rWdDtdeFRw8G4H+fWhF8bl3FAXIy0yjKywrux+HV\n0shId1KYB6btfrjd+bN3hO6pvr3aniYMMHOSs8nkQY+8WLnZ/ueV9CuI73uyloZJFQsaJm6hM6AS\n1T0FMLQol5PHlrJqWxXZGWkU5GTQrFCQk0lBTibLN+8DYFhx61QjmelpbNl7MLgTXqCFEgga4WMS\nftaWAEwdU8JvP3c0v7/imFav5cTQZXVMhCm9fnmtFzGmI1jQMHELXaAXb7dLuMunONNoJw7pGwwO\nfXplUtArk/3u4PZh/Xu3el9mWhqrth6aBhtoaRzqnmp50Z05abDvOl0wcRCnHdGf+Ted1uL5XlmR\ndysMd+zwQt67+XQG923dteaHNTRMqljQMHErdlsaJfnZCZ9KesphpYwbWMCZ4/szyE3nUdArM7je\nol/v7GCuqlAZ6RIMKiJONluRQ2MSoRsrvfo/J3smRmxLv7CFiTkZ/oNGQa9MSntnt/viH2v3VG4M\nAc2YaCxomLjlZmXQKzM9oeMZARnpabxw/TRmTRvFIPdXeUFOBgXuGMRhA1q3MuBQUMjPzmCYGxDy\nszKCqc1D11m0d6FdeJr0nEz/x8mL8yIeS8y44fQyekXZs92YWFjQMAlR0jvLcxZTIg0pDG1puEHD\no2sKDgWF0t7ZwfcF1mhAy9lTiWod5YRcmM89akCr179z5tjgfT+pVhLFNvgziWRpRExC3H7hUXGv\nPWhLoKXRp1dmcEB7bBstjeK8LI4dXsRbH+9mV8huf5khrYTMBKX0CA0an588nBeWbwecwHX3JRM5\neWwphw8oYNHGyrjPFZ6M8dLyIfxr4WbPskpsLZPe2RnBrj1jwllLwyTE9LGlHDk4+t7g8TrUPXWo\npXF4hKARaEkU52cFc1KF5pcKbWmE56Fqr9CgEbhInzCqmPduPp2Tx5YCcPq4/nz/7MPjPpeGNR9O\nPbxfG2X9fcbbLjyS750Tf/1M92VBw3QZgW6mvrmZTB5ZxLSyEsZG6J4K7OtdlJdNae9sbps5njsv\nOrTiO3T2VOJaGrEfx08LYGhRfN1+l5YP5eqpw9suiLM486Jj/M8kMz2PBQ3TZZTkZ/PwleVccuxQ\njh5WyF+/PKXFr/tQgdlFgTUkXzhhBJceNzT4eujgd6IyzrZoaSTkiI6po0paPTek0P9sr6FFuVx7\nahmrfnJ2m2VVldysDJ679iTP10+L0qIxPYMFDdOlnD6uP318LMSrcVdsF0cYZwlN+Bc+C6q9vFoa\nSvyj0D+5cDwjS/JaJDkc2Dd8PUzbn6FXVjob7jiPFT8+K2KZqWOcAHXUEO+uxj9efVzbFTbdmgUN\n0y0FNmsq9tgZEJKzorrFOo0EHj47I53Xv/MpprvjItB6M6ipY4p9Hy/adN/8GFKhmJ4ppUFDRL4j\nIioirdvfzutXicga93ZVR9fPdF2BzZqKI2zylIwpr5G6yqKZXlbadiEP4a2j0I2q2pLM6b55Wen8\n9cuTk3Z8k3opCxoiMhQ4A9gY4fUi4FZgCjAZuFVE4kvYY3qMwF7ixXmJX3AYSbZX91QbvVM/mjGe\nG04v83X80GN5bTvbXiM8cne1l2J5sbq7VLY0fgV8DyJ2+p4FzFHVPapaCcwB2h7JM4bQ7qnkrh0J\n1XIg3N+FMzM9jRtOH9t2wTCDouSsev07n+KF66b5Plb5iCLAO1twqN45rbuuLp8yrMVjW0jY/aUk\naIjIDGCLqi6NUmwwsCnk8Wb3Oa/jXSMiC0VkYUVFRQJrarqqwHauhbkdFzS8dvNr7zX0vKO89vZw\njnbBxEF87+zDWoxxhBpZkse4QQW+zxVoGLR1wX/qG1NbPTcsLGeXou3/0KZLSFrQEJFXRGSFx20m\ncDNwS1uH8HjO85+jqj6oquWqWl5a2r4+YtO9PPXNqdxy/rgO7SoJPdfo0jyAdq95uP/y1qnXA2ZM\nHEROZjqPfikxYwcl+dkU5mbyw/PHRSwzvDiXMf1ar4kJ/w8Z41bprXzzlNHxHSDM1VNHJPR4Jolp\nRFT1dK/nReQoYCSw1B2QGwIsFpHJqro9pOhm4FMhj4cAbySlsqbbGT+oD+MHJXeFerjQYYZ+BTls\nuOO85JwnwcfLykjj/VvOjFom0sr7VuJsaHz15NHc//raOI7QUvmIQh55e0PCjmdSkHtKVZcDwRVC\nIrIBKFfVXWFFXwJ+FjL4fSZwU4dU0ph2CKRd70iHD+jNlJFFcR2jrfGXP199HMdFOEdh2JqZWNel\nnDimmLc+3h3x9f4F2eyoqov4elv8ji0Z/zrVOg0RKReRhwFUdQ9wG/Cee/uJ+5wxnc5js6Z47iCY\nSIExh9AWzX9umM6PZx4Z13Hbmoh1yuH9PNdv3HnxBC45dmiL51Qjj42U5Gdx0TFDWjz3+cnR05tc\nd1oZn5s8LGqZRLGuLH9SHjRUdUSglaGqC1V1Vshrf1LVMe7tz6mrpTHeHp81hcdnTeHEMZ5LjXx7\n4AvHRhzYjuUYz3zzxJjf197f4peWD221XiRaO2PhD87gjHEt05CcN2EgH99+jmf5h64s5/Ipw4NB\nzXtyAMz97im+6ttWCvwfzRjv6zix6h/nfvCdjS3/NCYOU+MMFgFnjR/AJ7urmbs68uy/wMUzUsvg\nrPGt9/DwI94lH0MKe7G58iDg5K6K1kXl1QoJzTicH9LFF77bYKTjRmvh+f1s8W6KFU136yKzoGF6\nrPd/eEbC8k51hNs/fRSlvdcwrZ2ryCOJd4X4SzdMZ/WO/Xz6d2/TrDCwT/uz8qalCUtvOZMn39/M\n1NH+U6P4Ee1TPnNt7C00v44dXsjs5duSdvyOlvLuKWNSpTAviz69/KffSLX+BTn8/DNHJSyVe6Lk\nZWe0mKk2pl8+k4b2jekY937uaB6fNQWAPrmZfPHEkTEFszH98tssE+1wXtOJIxk30P8aGIBRpXkJ\nmx7dGXSuf33G9GCpWk3dnobGeRNaji+EH2N4jJMCZkwcFFdX37+/ekKbZT59dGL2CYn1h0ZOZjrT\nyhLTjdkZWNAwJoH69c5uNUOoMyvIyeCCCYM8XxvTL5+rTmg9u2nDHedx/+dbLj5M8xl5/KS1j8Qr\nKeN/v/spwGk1tuW2OGeZebnv80e3WWZw314duid8slnQMCaB3r35dH556cRUV8O3ZT86i6FF3q2C\nV759su/pvH4viVNHl/D7KKvdo/HaTnh4cZ7v92ckoVvv/AmDKO0dfXbUzEmtg3KvdmRE7iwsaBjT\nTV1aPoTxMeSgikf4D+loQeScCFNnk0GAUSV5/M8ZsSWFfOXbJwfvnz8hen3b6lb0amUM6pvDUR5B\nsCuw2VPGdDJfmTYyIce58+KOa/GEXxiT0R0Tem1e8L+nsd/dM6Utr33nUzGfK3Rg/b7PH8Pzy2a3\neP0zxwyOOQPAhZMG8fSSrTHXJZqsjDTqG5uDjzuiBWMtDWNMwows8e4uKvBIq+6XV/jpX5AT04yn\nRBKBey6dxG0XBrru/M1gCM+FloitgD973NC2CyWYtTSM6WQCv9JnX3dSTDvypdrjs6ZQ1t+5kIdf\n6Jf9KPK+5G1p76W1fHgh0+JcZd/ZXT11BDX1TTyxaDPgdHslm7U0jOmkxg/qE3GQujOaOqakzUHh\neMTa4fXE16d2yJ7nXmMaRw6OPpZ00zlHJOTcmelp3H3JRM4Y1x+Av3/l+IQcNxoLGsaYxEvCDNPA\ntbmojem1j39lSpv5qM49agBPtyNPl19//dKUiK/NOmkkp4/rzxVToidrfO7akyK+9tWTRwFQ4K4Z\neejKcjbccR79CqylYUyPMXPSYAb2yeHzHZTVtasIjT+vfPvkFjObvEwdXRIxH1Ug6WFpfnZw1Xpb\ns6MAbrvwSC451ll/c2m59zjCqYf383w+ks9OHhZ1z5VoG4h9/6zDWXrrmSnJaGBjGsZ0EgP65PDO\nTaeluhoJkawkfX7ShURz+ZRhzF6+jaOHOdv0rLn9HNIjzPR66YbpbK6sAeALxw+H44dz1yWtZ6QF\nWkChkwAyM9IozM3ksuNi+wFw7+eO5rq/v+8cIz3yd5iWJilLgWNBw5geaPLIooSl1ehKpo4p4c3v\nn8KQQqcl4pXHa+IQZ5bTYQN6c5jfHQs51CKaPraU/OyMNndD9DJj4iAG983hvx9VBCcVANxwehm/\nfmUN4OSySiULGsb0QP/ykaupuwoEDC+rfnI2GVF+4Xs576iB/HX+J5xYVsLDb67n6qnRxyoiCfyd\nHDu8iGOHOzslpkn8+64nmgUNY0zCJXJt32XHDeWxBRv5VBvTZ+d+9xQ2762J61y92rGvxo9mjOfb\nZ4ylMC8rrnGGyR5b6s759sms2LIvqbPSYmVBwxiTcIkc0ZgwpG/UAeOAYcW5Sd9y10t6mgQTJiZ6\nnGF0aT6jS51xnLdvPJWpd7wWMcFkR7GgYYxJmqmji6msaZ3u469fntylk/a1RyBdfHsH8wf17cXK\nH5+V8u/NgoYxJuEG9HHWC1x5wgjOPrL1NrSJ3n2wKzhzvLM2JDDQ3h55HbBYsS2pr4Exptv51qll\njOmXz1nj+6e6Kp2K146Gj8+aws79dSmoTfukdHGfiHxHRFREWm1rJSKTROQdEVkpIstE5LJU1NEY\nE7usjDRmThrcrTYfSpapY0q4sAtNf05ZS0NEhgJnABsjFKkBrlTVNSIyCFgkIi+p6t4Oq6QxxpgW\nUtnS+BXwPSIksVTV1aq6xr2/FdgJ9LyOUGOM6URSEjREZAawRVWX+iw/GcgC1kZ4/RoRWSgiCysq\nKhJYU2OMMaGS1j0lIq8AradNwM3A/wK+1tiLyEDgr8BVqtrsVUZVHwQeBCgvL+9k6yeNMab7SFrQ\nUNXTvZ4XkaOAkcBSd5BsCLBYRCar6vawsgXAbOAHqjo/WXU1xhjjT4cPhKvqciCYQ1hENgDlqror\ntJyIZAFPAY+q6r87tJLGGGM8dar9NESkXEQedh9eCkwHrhaRJe5tUgqrZ4wxPV7KF/ep6oiQ+wuB\nWe79vwF/S1G1jDHGeBD12uC2CxORCuCTOA5RAuxqs1THs3rFxuoVG6tXbLpjvYarapvLGrpd0IiX\niCxU1fJU1yOc1Ss2Vq/YWL1i05Pr1anGNIwxxnRuFjSMMcb4ZkGjtQdTXYEIrF6xsXrFxuoVmx5b\nLxvTMMYY45u1NIwxxvhmQcMYY4xvPT5oiMhdIvKhu9HTUyLSemstp9zZIvKRiHwsIjd2QL0ucTeg\nahaRiFPoRGSDiCx3V8wv7ET16ujvq0hE5ojIGvfPwgjlmkIyDDybxPpE/fwiki0i/3RfXyAiI5JV\nlxjrdbWIVIR8R7M6oE5/EpGdIrIiwusiIve6dV4mIscku04+6/UpEdkX8l3d0kH1Gioir4vIKvf/\n4vUeZZL3nalqj77hZNvNcO//AviFR5l0nLTso3BStC8FxiW5XkcAhwFv4OTmilRuA1DSgd9Xm/VK\n0fd1J3Cje/9Gr79H97UDHfAdtfn5gW8Af3Dvfxb4Zyep19XAfR3178k953TgGGBFhNfPBV4EBDge\nWNBJ6vUp4PmO/K7c8yCKDFgAAAVhSURBVA4EjnHv9wZWe/w9Ju076/EtDVV9WVUb3YfzcbLuhpsM\nfKyq61S1HvgHMDPJ9Vqlqh8l8xzt4bNeHf59ucf/i3v/L8CFST5fNH4+f2h9nwBOk+TvjZqKv5c2\nqepcYE+UIjNxEpeqOtmu+7pbJqS6XimhqttUdbF7fz+wCgjfLzZp31mPDxphvoQTncMNBjaFPN5M\n67+kVFHgZRFZJCLXpLoyrlR8X/1VdRs4/6kIyaQcJsfdsGu+iCQrsPj5/MEy7o+WfUBxkuoTS70A\nLnK7NJ5wt2VOtc78/+8EEVkqIi+KyPiOPrnbrXk0sCDspaR9ZylPWNgRom0IparPuGVuBhqBx7wO\n4fFc3HOV/dTLhxNVdauI9APmiMiH7i+kVNarw7+vGA4zzP2+RgGvichyVfXcETIOfj5/Ur6jNvg5\n53PA31W1TkS+htMaOjXJ9WpLKr4rPxbj5Gs6ICLnAk8DZR11chHJB/4PuEFVq8Jf9nhLQr6zHhE0\nNMKGUAEichVwPnCauh2CYTYDob+4hgBbk10vn8fY6v65U0SewumCiCtoJKBeHf59icgOERmoqtvc\nZvjOCMcIfF/rROQNnF9piQ4afj5/oMxmEckA+pD8rpA266Wqu0MePoQzzpdqSfn3FK/QC7WqviAi\nvxOREg3bGygZRCQTJ2A8pqpPehRJ2nfW47unRORs4PvADFWtiVDsPaBMREaKsznUZ4GkzbzxS0Ty\nRKR34D7OoL7nTI8Olorv61ngKvf+VUCrFpGIFIpItnu/BDgR+CAJdfHz+UPrezHwWoQfLB1ar7B+\n7xk4/eWp9ixwpTsj6HhgX6ArMpVEZEBgHEpEJuNcT3dHf1dCzivAH4FVqnpPhGLJ+846euS/s92A\nj3H6/pa4t8CMlkHACyHlzsWZpbAWp5sm2fX6NM6vhTpgB/BSeL1wZsEsdW8rO0u9UvR9FQOvAmvc\nP4vc58uBh937U4Hl7ve1HPhyEuvT6vMDP8H5cQKQA/zb/ff3LjAq2d+Rz3r93P23tBR4HTi8A+r0\nd2Ab0OD+2/oy8DXga+7rAtzv1nk5UWYTdnC9rg35ruYDUzuoXifhdDUtC7lundtR35mlETHGGONb\nj++eMsYY458FDWOMMb5Z0DDGGOObBQ1jjDG+WdAwxhjjmwUNY9pBRA7E+f4n3FXpiEi+iDwgImvd\nrKVzRWSKiGS593vEIlzTNVjQMKaDuTmK0lV1nfvUwzirwctUdTxOptkSdZIKvgpclpKKGuPBgoYx\ncXBX3N4lIivE2dfkMvf5NDetxEoReV5EXhCRi923XY67Yl1ERgNTgB+oajM4KU5UdbZb9mm3vDGd\ngjV7jYnPZ4BJwESgBHhPRObipCgZARyFk3F3FfAn9z0n4qw2BhgPLFHVpgjHXwEcl5SaG9MO1tIw\nJj4n4WSFbVLVHcB/cS7yJwH/VtVmVd2Ok5IjYCBQ4efgbjCpD+QYMybVLGgYE59IGydF21DpIE7u\nKXByF00UkWj/F7OB2nbUzZiEs6BhTHzmApeJSLqIlOJsEfou8CbOZkZpItIfZ2vQgFXAGAB19vJY\nCPw4JGNqmYjMdO8XAxWq2tBRH8iYaCxoGBOfp3CyjS4FXgO+53ZH/R9OZtQVwAM4O6vtc98zm5ZB\nZBbO5lIfi8hynH0sAnsfnAK8kNyPYIx/luXWmCQRkXx1dnUrxml9nKiq20WkF84Yx4lRBsADx3gS\nuEk74X7xpmey2VPGJM/zItIXyAJuc1sgqOpBEbkVZ8/mjZHe7G6U9LQFDNOZWEvDGGOMbzamYYwx\nxjcLGsYYY3yzoGGMMcY3CxrGGGN8s6BhjDHGt/8PFBRr5tD6JhUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x30ec5f28>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best C  is: [1.000e-02 1.000e-02 7.665e+01]\n"
     ]
    }
   ],
   "source": [
    "# dict with classes as the keys, and the values as the grid of scores obtained during cross-validating each fold,\n",
    "# Each dict value has shape (n_folds, len(Cs))\n",
    "n_Cs = len(Cs)\n",
    "n_classes = 3\n",
    "scores =  np.zeros((n_classes,n_Cs))\n",
    "print(np.mean(lr_cv_L2.scores_[1],axis = 0))\n",
    "for j in range(n_classes):\n",
    "    scores[j][:] = np.mean(lr_cv_L2.scores_[j],axis = 0)\n",
    "    \n",
    "mse_mean = np.mean(scores, axis = 0)\n",
    "plt.plot(np.log10(Cs), mse_mean.reshape(n_Cs,1)) \n",
    "#plt.plot(np.log10(reg.Cs)*np.ones(3), [0.28, 0.29, 0.30])\n",
    "plt.xlabel('log(C)')\n",
    "plt.ylabel('neg-logloss')\n",
    "plt.show()\n",
    "\n",
    "print ('best C  is:',lr_cv_L2.C_)  #对多类分类问题，每个类别的分类器有一个C"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_data_pred(lr_cv_L2,test)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 支持向量机分类回归问题"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#LinearSVC不能得到每类的概率，在Otto数据集要求输出每类的概率，这里只是示意SVM的使用方法\n",
    "#https://xacecask2.gitbooks.io/scikit-learn-user-guide-chinese-version/content/sec1.4.html\n",
    "#1.4.1.2. 得分与概率\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "SVC1 = LinearSVC().fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classification report for classifier LinearSVC(C=1.0, class_weight=None, dual=True, fit_intercept=True,\n",
      "     intercept_scaling=1, loss='squared_hinge', max_iter=1000,\n",
      "     multi_class='ovr', penalty='l2', random_state=None, tol=0.0001,\n",
      "     verbose=0):\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.00      0.00      0.00         7\n",
      "          1       0.37      0.54      0.43        28\n",
      "          2       0.71      0.65      0.68        65\n",
      "\n",
      "avg / total       0.57      0.57      0.56       100\n",
      "\n",
      "\n",
      "Confusion matrix:\n",
      "[[ 0  3  4]\n",
      " [ 0 15 13]\n",
      " [ 0 23 42]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    }
   ],
   "source": [
    "#在校验集上测试，估计模型性能\n",
    "y_predict = SVC1.predict(X_test)\n",
    "\n",
    "print(\"Classification report for classifier %s:\\n%s\\n\"\n",
    "      % (SVC1, metrics.classification_report(y_test, y_predict)))\n",
    "print(\"Confusion matrix:\\n%s\" % metrics.confusion_matrix(y_test, y_predict))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_data_pred(SVC1,test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 线性SVM正则参数调优"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "线性SVM LinearSVC的需要调整正则超参数包括C（正则系数，一般在log域（取log后的值）均匀设置候选参数）和正则函数penalty（L2/L1） \n",
    "\n",
    "采用交叉验证，网格搜索步骤与Logistic回归正则参数处理类似，在此略。\n",
    "\n",
    "这里我们用校验集（X_val、y_val）来估计模型性能"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def fit_grid_point_Linear(C, X_train, y_train, X_val, y_val):\n",
    "    \n",
    "    # 在训练集是那个利用SVC训练\n",
    "    SVC2 =  LinearSVC( C = C,penalty='l2')\n",
    "    SVC2 = SVC2.fit(X_train, y_train)\n",
    "    \n",
    "    # 在校验集上返回accuracy\n",
    "    accuracy = SVC2.score(X_val, y_val)\n",
    "    \n",
    "    print(\"accuracy: {}\".format(accuracy),\"C=%f\"%(C))\n",
    "    return accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy: 0.65 C=0.010000\n",
      "accuracy: 0.3 C=0.021544\n",
      "accuracy: 0.28 C=0.046416\n",
      "accuracy: 0.18 C=0.100000\n",
      "accuracy: 0.65 C=0.215443\n",
      "accuracy: 0.65 C=0.464159\n",
      "accuracy: 0.65 C=1.000000\n",
      "accuracy: 0.65 C=2.154435\n",
      "accuracy: 0.61 C=4.641589\n",
      "accuracy: 0.65 C=10.000000\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xl8XHW9//HXp03TUkopdIFKW9JC\naylyqxBqyVxZVUC0BUEWLyqg9AKyKN7LcrmiovcqoPwuYlU2EVFAhLKIlcpaZCk0QKGTLnSB0rSU\nhtIWCqXr5/fHdyZNwyQ5SebMmcm8n4/HPGbJycznZJK853y/3/P9mrsjIiIC0C3pAkREpHgoFERE\npJFCQUREGikURESkkUJBREQaKRRERKSRQkFERBopFEREpJFCQUREGlUkXUB7DRgwwKuqqpIuQ0Sk\npLzwwgtvu/vAtrYruVCoqqqitrY26TJEREqKmS2Jsp2aj0REpJFCQUREGikURESkkUJBREQaKRRE\nRKSRQkFERBopFEREpFHJnafQUc88A3/9K/zv/4JZ0tVIMdmyBa69FtasSboSaW7MGDjhBKgom/9U\nua1aBddcA6efDnvvHe9rlc2P+sUX4Wc/g7POgj33TLoaKSbPPAPf+164rQ8MxSO7fPzll8P3vw+n\nnFJ+4fD22yEMrrsO3n8f9tgj/lAom+ajVCpcP/10snVI8Umnw/XSpbB1qy7FctmyBaZMgd694etf\nh332gVtvhc2bk/19KYSGBrjkEqiqCh9mjzkGZs+Gc86J/7XLJhT22w/69FEoyEel09C3b/gUJsWj\nWzc47rhwlH/vveHv97TTYPRo+P3vu2Y4NDTAxRfD8OFw1VXwpS+F388774R99y1MDWUTChUVMH58\naCoQaSqdhk98Qk1HxapbNzj22BAO990XAvz00+HjH4dbboFNm5KusPNWroSLLgpHBldfDRMnQl0d\n3HFH6FcppLIJBQhNSK+8Au+9l3QlUizct4WCFDez8M/yhRfg/vuhXz8444wQDr/7XWmGw8qV8J//\nGY4MfvGLcGQ0Zw786U+huSwJZRUKNTWhrXLGjKQrkWLx1lvwzjsKhVJiBhMmQG1tGFG4667wzW+G\ncLjpptIIhxUrwuCGqqrQkfzlL4cw+OMfQ/NYksoqFMaPD4ei6leQrGwns0Kh9JjBF78IM2fCgw9C\n//5w5pkwahTceCNs3Jh0hR+1YgVceCGMGAH/93/wla/A3Llw220h1IpBWYVC376hw1mhIFnZUChU\nJ57kn1kYnfP88/C3v8GgQTBpUgiHG24ojnB480347ndDM9Evfwknngjz5oXRVKNGJV3d9soqFCD0\nK8yYEYa7iaTTMHBg+Ecipc0MvvCF8Pc9dSrsvjv8+7/DyJFw/fXJhMPy5XDBBeHI4Lrr4OSTQxj8\n/vehrmJUlqGwbl0Y8yuiTuauxwyOPhqefRYeegg+9rFw0uree8NvfgMbNsRfw7JlcP75IQwmT4av\nfhXmzw+jpeI++ayzyi4UamrCtZqQxD0M+1ModE1mcOSRYRj6tGkwZEg4+WvvveHXv44nHJYtg/PO\ng732CgF06qnw6qtw883hsVJQdqGw557hk4NCQd54Ixw1KhS6NjP4/OfD3/w//hH+B3z72+Gf9OTJ\n8OGHnX+NpUvDc44YAb/9LXztayEMbropPFZKyi4UzEITkkJB1MlcXszgc5+Df/4THnkkdPqee24I\nh1/9qmPhsHTptqOPG24IZ1wvWBBGPw0fnvddKIiyCwUIofDGG1Bfn3QlkiSFQnkygyOOgCefhEcf\nDaGQbfL55S9h/fq2n+ONN+Dss8P33HRTOMN64cLQoV1VFfsuxKpsQwE05UW5S6dDO3O/fklXIkkw\ng8MPh+nT4bHHwqf9Cy4I/+ivvTZ3OCxZsq3T+uabw0lzCxeGJqOuMvtyWYbC2LFh5kU1IZU3dTIL\nhHA47LAQDo8/Hk4i+853tp1gtn49vP76tuGtt9wSTpJbtCh0Jg8blvQe5FdZhkKPHjBunEKhnG3Z\nEqYVUChIU4ceGoLhiSfC3EPf/W74pz9yZDi3YNKkEAaTJ8PQoQkXG5OyDAUITUizZoXRJ1J+Fi0K\nQxLVnyC5HHJIaFJ68kk4+ODQf7BoUeiQHjIk6eriFWsomNlRZjbfzBaa2SUtbHOimc0xszozuz3O\neppKpcKnxeefL9QrSjHRnEcSxWc+A/fcEzqgu3oYZMUWCmbWHZgMHA2MAU4xszHNthkJXAqk3H1f\n4Dtx1dPcQQeFtkR1NpendDq8/0lNTyxSrOI8UhgHLHT3xe6+EbgTmNhsmzOBye6+GsDdV8ZYz3b6\n9QtNB+pXKE91daEjcccdk65EpLjEGQp7AEub3K/PPNbUKGCUmT1tZjPM7KhcT2Rmk8ys1sxqGxoa\n8lZgTU2YH2Xr1rw9pZQIzXkkklucoZBrcUNvdr8CGAkcCpwC3GRmHxk17u43uHu1u1cPHDgwbwWm\nUrB2bfjUKOVjw4YwBYE6mUU+Ks5QqAeaDtoaAizPsc397r7J3V8D5hNCoiCyJ7GpCam8vPpqWPRd\nRwoiHxVnKMwERprZcDOrBE4GHmi2zX3AYQBmNoDQnLQ4xpq2M2IE7LabOpvLjUYeibQstlBw983A\nucA0YC5wl7vXmdkVZjYhs9k0YJWZzQEeB/7T3VfFVVNzmhyvPNXVQUVF8Sx/KFJMKuJ8cnefCkxt\n9tjlTW47cGHmkoiaGpgyJayduvvuSVUhhZROhyUQKyuTrkSk+JTtGc1Z6lcoP+m0OplFWlL2obD/\n/tCrl0KhXLz/PixerP4EkZaUfShUVsKBB6qzuVzMnRuW4VQoiORW9qEAoQnpxRejLa4hpS17TopC\nQSQ3hQKhs3nTJpg5M+lKJG7pNPTsWTqLqIsUmkKBEAqgfoVykE6HSfC6d0+6EpHipFAA+veH0aMV\nCuVAcx6JtE6hkJFKhc5mTY7Xda1ZA/X1CgWR1igUMlIpWL0a5s9PuhKJy5w54VqhINIyhUKGTmLr\n+rJzHunENZGWKRQyRo6EAQMUCl1ZOg19+oSF2EUkN4VChlkYhaRQ6Lqy01t002+9SIv059FEKgUL\nFsDKgi0KKoWkkUcibVMoNJHtV3j22WTrkPxbuRIaGhQKIm1RKDRxwAFhLiQ1IXU92ekt1Mks0jqF\nQhO9eoVgUCh0PVptTSQahUIzqRTU1sKHHyZdieRTOg277qqFlETaolBoJpWCjRvhhReSrkTyKdvJ\nbJZ0JSLFTaHQTHZyPK2v0HW4hz4FNR2JtE2h0MygQeFENvUrdB3LlsHatepkFolCoZBDTU04UnBP\nuhLJB3Uyi0SnUMghlQpj2hcsSLoSyQfNeSQSnUIhB02O17Wk0zB4cFg3Q0Rap1DIYfRo2GUXhUJX\noU5mkegUCjl067atX0FK29atIRTUdCQSjUKhBTU1MHcuvPNO0pVIZ7z2GqxfryMFkagUCi3I9ivo\naKG0aeSRSPsoFFpw4IFQUaF+hVKXDYUxY5KtQ6RUKBRa0Ls37L+/QqHU1dVBVRXstFPSlYiUBoVC\nK1IpmDkzzIUkpSm72pqIRKNQaEUqFWZLfemlpCuRjti0CebNU3+CSHsoFFqRnRxPTUilacGCEAwK\nBZHoFAqtGDwYhg9XKJQqjTwSab9YQ8HMjjKz+Wa20MwuyfH108yswcxmZS7firOejkilQihocrzS\nU1cXTkQcPTrpSkRKR2yhYGbdgcnA0cAY4BQzyzUw8M/u/snM5aa46umoVAreegsWL066EmmvdBr2\n3jsssyoi0cR5pDAOWOjui919I3AnMDHG14uFTmIrXdnV1kQkujhDYQ9gaZP79ZnHmjvezF4xs7vN\nbGiuJzKzSWZWa2a1DQ0NcdTaojFjoG9f9SuUmvXrYeFChYJIe8UZCrlWw23eMv9XoMrd/wV4BLg1\n1xO5+w3uXu3u1QMHDsxzma3r3h0OOkihUGrmzQuT4SkURNonzlCoB5p+8h8CLG+6gbuvcvcNmbs3\nAgfEWE+HpVKh03LNmqQrkajq6sK1QkGkfeIMhZnASDMbbmaVwMnAA003MLPBTe5OAObGWE+HpVJh\n9NGzzyZdiUSVTkOPHqGjWUSiiy0U3H0zcC4wjfDP/i53rzOzK8xsQmaz882szsxeBs4HTourns4Y\nNy40I6mzuXSk02Eoao8eSVciUloq4nxyd58KTG322OVNbl8KXBpnDfnQpw+MHat+hVKSTm87I11E\notMZzRGlUvDcc2HaBClu770HS5aoP0GkIxQKEaVS8MEH8PLLSVcibZkzJ1wrFETaT6EQUfYkNjUh\nFb/snEeaMluk/RQKEQ0ZAsOGqbO5FKTTsMMOYTJDEWkfhUI71NRocrxSkF1Yp5t+u0XaTX827ZBK\nwbJl8MYbSVcirdGcRyIdp1BoB/UrFL9Vq2DFCoWCSEcpFNphv/3COQsKheKVnd5CncwiHaNQaIeK\nChg/XqFQzLTamkjnKBTaKZWC2bPh3XeTrkRySadh551hj1yTtItImyKFgpndY2bHmFnZh0hNTZiS\n+bnnkq5EcqmrC0cJlmvidhFpU9R/8r8BvgosMLOfmVnZrno7fnwY6qgmpOLjrpFHIp0VKRTc/RF3\n/zdgf+B14GEze8bMTjezspqHsm/f0OGsUCg+K1bAO++ok1mkMyI3B5lZf8LU1t8CXgKuJYTEw7FU\nVsRSKZgxAzZvTroSaUqdzCKdF7VPYQrwT6A38CV3n+Duf3b384A+cRZYjFIpWLdu2z8hKQ4KBZHO\ni7qewq/c/bFcX3D36jzWUxKy8/Q//TR88pPJ1iLb1NXBoEFQ4GW8RbqUqM1H+5hZv+wdM9vFzM6J\nqaait+ee8LGPqV+h2GTnPBKRjosaCme6e+Oy9e6+GjgznpKKn1loQlIoFI+tW7cNRxWRjosaCt3M\nto38NrPuQGU8JZWGVCpMjFdfn3QlAuG9WLdOoSDSWVFDYRpwl5kdYWaHA3cAD8VXVvHLTo6n9RWK\ngzqZRfIjaihcDDwGnA18G3gUuCiuokrB2LHQu7eakIqFJsITyY9Io4/cfSvhrObfxFtO6ejRA8aN\nUygUi3Q6rI63885JVyJS2qKepzDSzO42szlmtjh7ibu4YpdKwaxZoS1bkqXpLUTyI2rz0S2Eo4TN\nwGHAH4Db4iqqVKRSsGULPP980pWUt82bYe5chYJIPkQNhR3c/VHA3H2Ju/8QODy+skrDQQeFazUh\nJWvRItiwQaEgkg9Rz2j+MDNt9gIzOxdYBgyKr6zS0K9f6NjUCKRkZTuZFQoinRf1SOE7hHmPzgcO\nAE4FvhFXUaUklYJnnw0nT0ky0ulwQuE++yRdiUjpazMUMieqneju69y93t1Pd/fj3X1GAeoreqkU\nrF277dOqFF46DSNGhCHCItI5bYaCu28BDmh6RrNskz2JTf0KydHII5H8idp89BJwv5l9zcy+nL3E\nWVipGDECdttNoZCUDRvg1VcVCiL5ErWjeVdgFduPOHJgSt4rKjHZyfHU2ZyMV18Nw4IVCiL5EfWM\n5tPjLqSU1dTAlClhOcjdd0+6mvKSnfNI01uI5EekUDCzWwhHBttx9zPyXlEJatqvcPzxydZSbtJp\nqKiAj3886UpEuoaofQoPAn/LXB4F+gJtTu5gZkeZ2XwzW2hml7Sy3Qlm5mZWkqu47b8/9OqlfoUk\npNMwahRUlvVE7iL5E7X56J6m983sDuCR1r4nM5R1MvA5oB6YaWYPuPucZtvtRDj/4bl21F1UKivh\nwAMVCklIp6G6JD9KiBSnqEcKzY0EhrWxzThgobsvdveNwJ3AxBzb/Ri4Cviwg7UUhVQKXnwR1q9P\nupLy8f778Npr6mQWyaeos6S+Z2bvZi/AXwlrLLRmD2Bpk/v1mceaPu+ngKHu/mAbrz/JzGrNrLah\noSFKyQVXUxMmZps5M+lKysfcueCuTmaRfIoUCu6+k7v3bXIZ1bxJKYdcJ7s1dlZn5lL6f8D3Irz+\nDe5e7e7VAwcOjFJywdXUhGs1IRWOVlsTyb+oRwrHmdnOTe73M7Nj2/i2emBok/tDgOVN7u8EfAJ4\nwsxeB8YDD5RqZ3P//jB6tEKhkNJp6NkT9tor6UpEuo6ofQo/cPe12Tvuvgb4QRvfMxMYaWbDzawS\nOBl4oMlzrHX3Ae5e5e5VwAxggrvXtmsPikj2JDZNjlcY6TSMGQPduyddiUjXETUUcm3X6sgld98M\nnAtMA+YCd7l7nZldYWYT2ldmaUilYPVqmDcv6UrKQ12dmo5E8i3qNBe1ZnYNYYipA+cBL7T1Te4+\nFZja7LHLW9j20Ii1FK1sv8Izz4RPsBKfNWugvl6dzCL5FvVI4TxgI/Bn4C5gPfDtuIoqVaNGwYAB\n6lcoBC2sIxKPqCevvQ+0eEayBGbhaEGhED+NPBKJR9TRRw+bWb8m93cxs2nxlVW6UilYsABWrky6\nkq4tnYY+fWBYW6dQiki7RG0+GpAZcQSAu69GazTnlJ0cT1Npxyvbyayln0TyK2oobDWzxs9kZlZF\njllTBQ44IMyFpFCIVzqtTmaROEQdfXQZ8JSZTc/cPxiYFE9Jpa1XrxAM6leIz8qV0NCg/gSROESd\n5uIhoBqYTxiB9D3CCCTJIZWC2lr4sKSn+Cte6mQWiU/UjuZvEdZR+F7mchvww/jKKm2pFGzcCC+0\neSaHdIRCQSQ+UfsULgAOBJa4+2HAp4DinK60CGhyvHjV1YW5pnbbLelKRLqeqKHwobt/CGBmPd19\nHqAFEFswaBCMHKnO5rhkO5k18kgk/6KGQn3mPIX7gIfN7H62n/FUmqmpCaHgGqOVV+4hFNR0JBKP\nqB3Nx7n7Gnf/IfB94Gagramzy1oqFUbILFiQdCVdS309vPuuQkEkLlGHpDZy9+ltbyXZk9iefjrM\niST5oU5mkXh1dI1macPo0bDLLupszrfsRHg6cU0kHgqFmHTrpsnx4pBOw+DBsOuuSVci0jUpFGJU\nUxMW3Fm1KulKug51MovES6EQo2y/wrPPJltHV7FlC8yZo1AQiZNCIUYHHggVFWpCypfXXoP16xUK\nInFSKMSod2/Yf3+FQr5otTWR+CkUYpZKwcyZYS4k6ZzscFStfy0SH4VCzGpqwmypL72UdCWlL52G\nqqqw4pqIxEOhELOmJ7FJ52jkkUj8FAoxGzwYhg9XKHTWpk0wf75CQSRuCoUCSKVCKGhyvI5bsCAE\ng85kFomXQqEAUil46y1YvDjpSkqX5jwSKQyFQgFk+xW0vkLHpdNh6pDRo5OuRKRrUygUwJgx0Lev\n+hU6I50OCxf16pV0JSJdm0KhALp3h4MOUih0hkYeiRSGQqFAUqlwRu6aNUlXUnrWr4dFi9TJLFII\nCoUCSaXC6CNNjtd+8+bB1q06UhApBIVCgYwbF5qR1ITUfhp5JFI4CoUC6dMHxo7VCKSOSKehshL2\n3jvpSkS6PoVCAaVS8Nxz4SQsiS6dDkNRe/RIuhKRri/WUDCzo8xsvpktNLNLcnz9LDObbWazzOwp\nM+vS81+mUvDBB/Dyy0lXUlrq6tTJLFIosYWCmXUHJgNHA2OAU3L807/d3fdz908CVwHXxFVPMdDk\neO337ruwZIn6E0QKJc4jhXHAQndf7O4bgTuBiU03cPd3m9zdEejSswMNGQLDhsEjj2h9hajmzAnX\nCgWRwqiI8bn3AJY2uV8PfLr5Rmb2beBCoBI4PNcTmdkkYBLAsGHD8l5oIR12GNx6K/TrF9ZaOOQQ\nOPTQMDqpZ8+kqys+GnkkUlhxHilYjsc+ciTg7pPdfS/gYuC/cz2Ru9/g7tXuXj1w4MA8l1lYkyfD\nlClw5pnw9ttw+eVw8MEhJA4/HK64Ap58MizMIyEUevcOi+uISPziPFKoB4Y2uT8EWN7K9ncCv4mx\nnqKw445w3HHhArBqFfzznzB9OjzxBPzwh+Ekt549w9QYhx4ajibGjy/PeX/q6sLcUd00Tk6kIOIM\nhZnASDMbDiwDTga+2nQDMxvp7gsyd48BFlBm+veHY48NF4DVq0NIPPFEuPzoR9tCYvz4bc1N48fD\nDjskWHiBpNNw1FFJVyFSPmILBXffbGbnAtOA7sDv3L3OzK4Aat39AeBcM/sssAlYDXwjrnpKxS67\nwIQJ4QIhJJ56KgTE9Onwk5+EJqbKyu1D4qCDul5IvP02rFih/gSRQjIvseXAqqurvba2NukyErN2\n7baQeOIJePHFMC9QZWXorD700G0h0bt3srV21vTpYV8eegiOPDLpakRKm5m94O7VbW0XZ/ORxGDn\nneGYY8IFQkg8/fS2I4mf/jQcTfTosS0kDjkkjHTaccckK28/jTwSKTwdKXQx774bQiLbcV1bC1u2\nQEVFCIlsc1NNTZiPqZidcw7cfntoQrNcY9lEJLKoRwoKhS7uvffCJHzZ5qbaWti8OYTE+efDz39e\nvP9wDz44NI099VTSlYiUPjUfCQA77RTa47Nt8uvWhZC47Ta45pow8+jZZydbYy7uofnoxBOTrkSk\nvCgUykyfPvD5z8MRR8A774SjhU98Aj7zmaQr296bb4ZmI/UniBSWTgkqU927w5/+BMOHwwknQH19\n0hVtT53MIslQKJSxfv3g/vvDGsjHHVdcU2vU1YVrTZktUlgKhTK3zz6hf6G2Fs46K7TlF4N0GgYN\nghKf6kqk5CgUhIkTw5xLt94K112XdDVBOq2mI5EkKBQEgO9/P4TDhRfC448nW8vWraH5SKEgUngK\nBQHCLKR/+AOMHBmGgS5ZklwtS5bA++8rFESSoFCQRn37ho7njRtDx/MHHyRThzqZRZKjUJDtjBoV\nppaYNSssBJREx3N2OKpCQaTwFAryEcccEybVu/32cNZzoaXTMHRomPxPRApLoSA5XXppOKntoovg\n4YcL+9oaeSSSHIWC5GQGt9wSlsI86SRYvLgwr7t5M8ydq1AQSYpCQVrUpw/cd1+4feyxYURQ3BYt\nCh3d6k8QSYZCQVq1115wxx1hRNDpp8ff8aw5j0SSpVCQNh15JPzsZ/CXv8CVV8b7Wul0aLraZ594\nX0dEclMoSCT/8R9w8snwX/8Ff/97fK+TToejk1JfX1qkVCkUJBIzuPlmGDsWTjkFFiyI53U08kgk\nWQoFiax3b7j33rCU58SJYanPfNqwIYSNOplFkqNQkHapqoK77oJXX4Wvfz1MXpcv8+fDli06UhBJ\nkkJB2u3ww+HnPw/DVf/nf/L3vBp5JJI8hYJ0yAUXwNe+BpdfDg88kJ/nTKdD09SoUfl5PhFpP4WC\ndIgZXH89HHAAnHoqzJvX+edMp0MgVFZ2/rlEpGMUCtJhO+wQOp579Qodz2vXdu75tLCOSPIUCtIp\nQ4fC3XeHuZFOPbXjHc/vvx+eQ6EgkiyFgnTawQfDtdfCgw+GtZ47Ys6ccK1QEEmWQkHy4uyz4Ywz\n4Mc/hilT2v/9GnkkUhwUCpIXZjB5Mnz60+H8hew/+ajS6dA3MWJEPPWJSDQKBcmbXr3gnntgp53C\nVNurV0f/3rq6MAle9+7x1ScibVMoSF7tsUcIhjfeCHMkbdkS7fs055FIcYg1FMzsKDObb2YLzeyS\nHF+/0MzmmNkrZvaome0ZZz1SGDU1oSlp2jS47LK2t1+9GpYtUyiIFIPYQsHMugOTgaOBMcApZjam\n2WYvAdXu/i/A3cBVcdUjhXXmmXDWWWH9hbvuan3burpwrVAQSV6cRwrjgIXuvtjdNwJ3AhObbuDu\nj7v7B5m7M4AhMdYjBXbttZBKhRXbXn655e2yndKaHVUkeXGGwh7A0ib36zOPteSbQIzLt0ihVVaG\nE9v69Qsdz6tW5d6uri6sBz1sWGHrE5GPijMULMdjOVf4NbNTgWrg6ha+PsnMas2stqGhIY8lStx2\n3z1MhbF8OZx0Emze/NFtsp3Mlus3RkQKKs5QqAeGNrk/BFjefCMz+yxwGTDB3TfkeiJ3v8Hdq929\neuDAgbEUK/EZNw5++1t49FG4+OLtv+YOs2erP0GkWMQZCjOBkWY23MwqgZOB7SZZNrNPAdcTAmFl\njLVIwk4/Hc47D665Bv74x22Pr1wZmpUUCiLFIbZQcPfNwLnANGAucJe715nZFWY2IbPZ1UAf4C9m\nNsvM8jQzvxSjX/wCDjkkjEx68cXwmDqZRYpLRZxP7u5TganNHru8ye3Pxvn6Ulx69AjDU6urQ8dz\nba2Go4oUG53RLAU1aFDoeG5ogBNPhJdegv79Ybfdkq5MREChIAk44AC48UaYPh1uvVUjj0SKiUJB\nEnHqqXDhhWH0kZqORIpHrH0KIq258sqwpOfxxyddiYhkKRQkMRUV8JOfJF2FiDSl5iMREWmkUBAR\nkUYKBRERaaRQEBGRRgoFERFppFAQEZFGCgUREWmkUBARkUbmnnMxtKJlZg3Akg5++wDg7TyWkyTt\nS/HpKvsB2pdi1Zl92dPd21ylrORCoTPMrNbdq5OuIx+0L8Wnq+wHaF+KVSH2Rc1HIiLSSKEgIiKN\nyi0Ubki6gDzSvhSfrrIfoH0pVrHvS1n1KYiISOvK7UhBRERa0aVDwcyuNrN5ZvaKmd1rZv1a2O4o\nM5tvZgvN7JJC1xmFmX3FzOrMbKuZtTj6wMxeN7PZZjbLzGoLWWNU7diXon5fzGxXM3vYzBZkrndp\nYbstmfdjlpk9UOg6W9PWz9jMeprZnzNff87MqgpfZTQR9uU0M2to8l58K4k622JmvzOzlWaWbuHr\nZma/zOznK2a2f14LcPcuewE+D1Rkbl8JXJljm+7AImAEUAm8DIxJuvYcde4DfBx4AqhuZbvXgQFJ\n19vZfSmF9wW4Crgkc/uSXL9fma+tS7rWjv6MgXOA32Zunwz8Oem6O7EvpwG/SrrWCPtyMLA/kG7h\n618A/g4YMB54Lp+v36WPFNz9H+6+OXN3BjAkx2bjgIXuvtjdNwJ3AhMLVWNU7j7X3ecnXUc+RNyX\nUnhfJgK3Zm7fChybYC0dEeVn3HQf7waOMDMrYI1RlcLvSyTu/iTwTiubTAT+4MEMoJ+ZDc7X63fp\nUGjmDEK6NrcHsLTJ/frMY6XKgX+Y2QtmNinpYjqhFN6X3dz9TYDM9aAWtutlZrVmNsPMiik4ovyM\nG7fJfMBaC/QvSHXtE/X35fhgX/WIAAAEJklEQVRMk8vdZja0MKXlXax/GyW/RrOZPQLsnuNLl7n7\n/ZltLgM2A3/K9RQ5HktkSFaUfYkg5e7LzWwQ8LCZzct88iioPOxLUbwvre1HO55mWOY9GQE8Zmaz\n3X1RfirslCg/46J4HyKIUudfgTvcfYOZnUU4Ajo89sryL9b3pORDwd0/29rXzewbwBeBIzzTINdM\nPdD0E8MQYHn+KoyurX2J+BzLM9crzexewmF1wUMhD/tSFO9La/thZm+Z2WB3fzNz+L6yhefIvieL\nzewJ4FOE9u+kRfkZZ7epN7MKYGdab9pISpv74u6rmty9kdDPWIpi/dvo0s1HZnYUcDEwwd0/aGGz\nmcBIMxtuZpWEzrSiGiESlZntaGY7ZW8TOtpzjmAoAaXwvjwAfCNz+xvAR46AzGwXM+uZuT0ASAFz\nClZh66L8jJvu4wnAYy18uEpam/vSrN19AjC3gPXl0wPA1zOjkMYDa7PNmHmRdE97zL34Cwltb7My\nl+woio8BU5v15r9K+PR2WdJ1t7AvxxE+IWwA3gKmNd8XwsiLlzOXulLel1J4Xwht648CCzLXu2Ye\nrwZuytyuAWZn3pPZwDeTrrvZPnzkZwxcQfggBdAL+Evmb+l5YETSNXdiX36a+bt4GXgcGJ10zS3s\nxx3Am8CmzN/JN4GzgLMyXzdgcmY/Z9PKaMSOXHRGs4iINOrSzUciItI+CgUREWmkUBARkUYKBRER\naaRQEBGRRgoFkRzMbF0nv//uzBnMmFkfM7vezBZlZod90sw+bWaVmdslfxKpdB0KBZE8M7N9ge7u\nvjjz0E2Es4BHuvu+hNk6B3iYuO1R4KREChXJQaEg0orMWaNXm1k6s07FSZnHu5nZrzOf/B80s6lm\ndkLm2/6NzNnNZrYX8Gngv919K4TpLtz9b5lt78tsL1IUdNgq0rovA58ExgIDgJlm9iRhuooqYD/C\n7Khzgd9lvidFOCsVYF9glrtvaeH508CBsVQu0gE6UhBp3b8SZtbc4u5vAdMJ/8T/FfiLu2919xWE\naROyBgMNUZ48ExYbs3NWiSRNoSDSupYWlGltoZn1hDmDIMy1M9bMWvtb6wl82IHaRPJOoSDSuieB\nk8ysu5kNJCyV+DzwFGHBlm5mthtwaJPvmQvsDeBh3YRa4EfZFcvMbKSZTczc7g80uPumQu2QSGsU\nCiKtuxd4hTCz5mPARZnmonsIM1imgeuB5wirkgH8je1D4luEhXoWmtlswlz+2fnvDwOmxrsLItFp\nllSRDjKzPu6+LvNp/3nCqncrzGwHQh9DqpUO5uxzTAEu9S6y/raUPo0+Eum4B82sH1AJ/DhzBIG7\nrzezHxDWzX2jpW/OLAZznwJBiomOFEREpJH6FEREpJFCQUREGikURESkkUJBREQaKRRERKSRQkFE\nRBr9f25bShECMGvyAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x305e9828>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#需要调优的参数\n",
    "C_s = np.logspace(-2, 1, 10)# logspace(a,b,N)把10的a次方到10的b次方区间分成N份  \n",
    "#penalty_s = ['l1','l2']\n",
    "\n",
    "accuracy_s = []\n",
    "for i, oneC in enumerate(C_s):\n",
    "#    for j, penalty in enumerate(penalty_s):\n",
    "    tmp = fit_grid_point_Linear(oneC, X_train, y_train, X_test, y_test)\n",
    "    accuracy_s.append(tmp)\n",
    "\n",
    "x_axis = np.log10(C_s)\n",
    "#for j, penalty in enumerate(penalty_s):\n",
    "plt.plot(x_axis, np.array(accuracy_s), 'b-')\n",
    "    \n",
    "plt.legend()\n",
    "plt.xlabel( 'log(C)' )                                                                                                      \n",
    "plt.ylabel( 'accuracy' )\n",
    "plt.savefig('SVM_Otto.png' )\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RBF核SVM正则参数调优\n",
    "\n",
    "RBF核是SVM最常用的核函数。\n",
    "RBF核SVM 的需要调整正则超参数包括C（正则系数，一般在log域（取log后的值）均匀设置候选参数）和核函数的宽度gamma\n",
    "C越小，决策边界越平滑； \n",
    "gamma越小，决策边界越平滑。\n",
    "\n",
    "采用交叉验证，网格搜索步骤与Logistic回归正则参数处理类似，在此略。\n",
    "\n",
    "这里我们用校验集（X_val、y_val）来估计模型性能"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.svm import SVC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def fit_grid_point_RBF(C, gamma, X_train, y_train, X_val, y_val):\n",
    "    \n",
    "    # 在训练集是那个利用SVC训练\n",
    "    SVC3 =  SVC( C = C, kernel='rbf', gamma = gamma)\n",
    "    SVC3 = SVC3.fit(X_train, y_train)\n",
    "    \n",
    "    # 在校验集上返回accuracy\n",
    "    accuracy = SVC3.score(X_val, y_val)\n",
    "    \n",
    "    print(\"accuracy: {}\".format(accuracy),\"C=%f\"%(C),\"gamma=%f\"%(gamma))\n",
    "    return accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy: 0.65 C=0.010000 gamma=0.010000\n",
      "accuracy: 0.65 C=0.010000 gamma=0.100000\n",
      "accuracy: 0.65 C=0.010000 gamma=1.000000\n",
      "accuracy: 0.65 C=0.010000 gamma=10.000000\n",
      "accuracy: 0.65 C=0.010000 gamma=100.000000\n",
      "accuracy: 0.65 C=0.021544 gamma=0.010000\n",
      "accuracy: 0.65 C=0.021544 gamma=0.100000\n",
      "accuracy: 0.65 C=0.021544 gamma=1.000000\n",
      "accuracy: 0.65 C=0.021544 gamma=10.000000\n",
      "accuracy: 0.65 C=0.021544 gamma=100.000000\n",
      "accuracy: 0.65 C=0.046416 gamma=0.010000\n",
      "accuracy: 0.65 C=0.046416 gamma=0.100000\n",
      "accuracy: 0.65 C=0.046416 gamma=1.000000\n",
      "accuracy: 0.65 C=0.046416 gamma=10.000000\n",
      "accuracy: 0.65 C=0.046416 gamma=100.000000\n",
      "accuracy: 0.65 C=0.100000 gamma=0.010000\n",
      "accuracy: 0.65 C=0.100000 gamma=0.100000\n",
      "accuracy: 0.65 C=0.100000 gamma=1.000000\n",
      "accuracy: 0.65 C=0.100000 gamma=10.000000\n",
      "accuracy: 0.65 C=0.100000 gamma=100.000000\n",
      "accuracy: 0.65 C=0.215443 gamma=0.010000\n",
      "accuracy: 0.65 C=0.215443 gamma=0.100000\n",
      "accuracy: 0.65 C=0.215443 gamma=1.000000\n",
      "accuracy: 0.65 C=0.215443 gamma=10.000000\n",
      "accuracy: 0.65 C=0.215443 gamma=100.000000\n",
      "accuracy: 0.65 C=0.464159 gamma=0.010000\n",
      "accuracy: 0.65 C=0.464159 gamma=0.100000\n",
      "accuracy: 0.65 C=0.464159 gamma=1.000000\n",
      "accuracy: 0.65 C=0.464159 gamma=10.000000\n",
      "accuracy: 0.65 C=0.464159 gamma=100.000000\n",
      "accuracy: 0.65 C=1.000000 gamma=0.010000\n",
      "accuracy: 0.65 C=1.000000 gamma=0.100000\n",
      "accuracy: 0.65 C=1.000000 gamma=1.000000\n",
      "accuracy: 0.65 C=1.000000 gamma=10.000000\n",
      "accuracy: 0.65 C=1.000000 gamma=100.000000\n",
      "accuracy: 0.65 C=2.154435 gamma=0.010000\n",
      "accuracy: 0.65 C=2.154435 gamma=0.100000\n",
      "accuracy: 0.65 C=2.154435 gamma=1.000000\n",
      "accuracy: 0.65 C=2.154435 gamma=10.000000\n",
      "accuracy: 0.65 C=2.154435 gamma=100.000000\n",
      "accuracy: 0.65 C=4.641589 gamma=0.010000\n",
      "accuracy: 0.65 C=4.641589 gamma=0.100000\n",
      "accuracy: 0.65 C=4.641589 gamma=1.000000\n",
      "accuracy: 0.65 C=4.641589 gamma=10.000000\n",
      "accuracy: 0.65 C=4.641589 gamma=100.000000\n",
      "accuracy: 0.65 C=10.000000 gamma=0.010000\n",
      "accuracy: 0.65 C=10.000000 gamma=0.100000\n",
      "accuracy: 0.65 C=10.000000 gamma=1.000000\n",
      "accuracy: 0.65 C=10.000000 gamma=10.000000\n",
      "accuracy: 0.65 C=10.000000 gamma=100.000000\n"
     ]
    }
   ],
   "source": [
    "#需要调优的参数\n",
    "C_s = np.logspace(-2, 1, 10)# logspace(a,b,N)把10的a次方到10的b次方区间分成N份 \n",
    "gamma_s = np.logspace(-2, 2, 5)    \n",
    "\n",
    "accuracy_s = []\n",
    "for i, oneC in enumerate(C_s):\n",
    "    for j, gamma in enumerate(gamma_s):\n",
    "        tmp = fit_grid_point_RBF(oneC, gamma, X_train, y_train, X_test, y_test)\n",
    "        accuracy_s.append(tmp)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "从上述部分运行结果来看，gamma参数设置不合适（gamma越大，对应RBF核的sigma越小，决策边界更复杂，可能发生了过拟合）\n",
    "所以调小gamma值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy: 0.65 C=0.100000 gamma=0.000000\n",
      "accuracy: 0.65 C=0.278256 gamma=0.000000\n",
      "accuracy: 0.65 C=0.774264 gamma=0.000000\n",
      "accuracy: 0.64 C=2.154435 gamma=0.000000\n",
      "accuracy: 0.66 C=5.994843 gamma=0.000000\n",
      "accuracy: 0.66 C=16.681005 gamma=0.000000\n",
      "accuracy: 0.63 C=46.415888 gamma=0.000000\n",
      "accuracy: 0.62 C=129.154967 gamma=0.000000\n",
      "accuracy: 0.54 C=359.381366 gamma=0.000000\n",
      "accuracy: 0.52 C=1000.000000 gamma=0.000000\n"
     ]
    }
   ],
   "source": [
    "#需要调优的参数\n",
    "C_s = np.logspace(-1, 3, 10)# logspace(a,b,N)把10的a次方到10的b次方区间分成N份 \n",
    "gamma_s = np.logspace(-8, -8, 1)  \n",
    "\n",
    "accuracy_s = []\n",
    "for i, oneC in enumerate(C_s):\n",
    "    for j, gamma in enumerate(gamma_s):\n",
    "        tmp = fit_grid_point_RBF(oneC, gamma, X_train, y_train, X_test, y_test)\n",
    "        accuracy_s.append(tmp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEKCAYAAADjDHn2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xl8VNX5+PHPk42wB8giECAgSxIW\nA4SwqQVCK3UBFyygIPhTqV+l2vZbWu3XVku17m1ttS5FvwJakbogKhXZFEVAgoQtYQloJIAkIDsE\nsjy/P+aS7xCSzCTkZrI879drXsy9c+69z1wy88w9555zRFUxxhhjKhIU6ACMMcbUfpYsjDHG+GTJ\nwhhjjE+WLIwxxvhkycIYY4xPliyMMcb4ZMnCGGOMT5YsjDHG+GTJwhhjjE8hgQ6gukRGRmpcXFyg\nwzDGmDpl3bp1B1Q1yle5epMs4uLiSEtLC3QYxhhTp4hItj/lrBrKGGOMT5YsjDHG+GTJwhhjjE+W\nLIwxxvhkycIYY4xPriYLERklIttEJEtE7iunzE9EJENEtojIv7zWdxSRj0Uk03k9zs1YjTHGlM+1\nW2dFJBh4DvghkAOsFZEFqprhVaYbcD8wVFUPiUi01y5mA4+o6mIRaQYUuxWrMcaYirnZzyIFyFLV\nXQAiMhcYA2R4lbkDeE5VDwGoaq5TNhEIUdXFzvrjLsZpTLmWZOxnY87hQIdBUJBwQ79YOrRuEuhQ\nTAPlZrJoD+z2Ws4BBpYq0x1ARFYCwcBDqvqRs/6wiLwDdAaWAPepapH3xiIyFZgK0LFjRzfeg2mg\nioqVJxZt5cVPdwEgEth4VOHd9XtYcPeltGwSGthgTIPkZrIo6+OlZRy/GzAMiAU+E5FezvrLgL7A\nt8CbwBTg5XN2pvoS8BJAcnJy6X0bUyUnThdy79x0lmTuZ+Kgjjx4TU9CgwN7L8i67O8Z/9Jq7n1z\nPS9PHkBwUICzl2lw3PwE5AAdvJZjgb1llHlPVQtU9WtgG57kkQOsV9VdqloIzAf6uRirMQDkHDrJ\nDc9/wfJtucwY05OHr+0d8EQB0L9Tax4a3ZNPtuXxl8XbAx2OaYDc/BSsBbqJSGcRCQPGAwtKlZkP\nDAcQkUg81U+7nG1bicjZwa1GcG5bhzHVbl32Ia59biV7Dp/i1VsHcMvguECHdI6bB3ZiQkoHnl2e\nxUeb9wU6HNPAuJYsnCuCacAiIBOYp6pbRGSGiIx2ii0CDopIBrAcmK6qB522iV8BS0VkE54qrX+6\nFasx73yVw4SXVtOsUQjv3jWUy7r5HIQzIB4a3ZO+HSP45bwNbN9/LNDhmAZEVOtHVX9ycrLaqLOm\nsoqLlac+3sY/PtnJ4C5teH5iPyKahAU6rArtP5rP1X//nGaNQph/91BaNrYGb1N1IrJOVZN9lQt8\nZawxAXLidCF3vraOf3yykwkpHZl9W0qtTxQAMS3Cef7mfuz+/iS/eDOd4uL68YPP1G6WLEyDtPfw\nKW58YRVLMvfz4DWJ/Om6XrWiIdtfyXGteXB0T5ZtzeWvS6zB27iv3kx+ZIy/vvr2EFNnr+N0QRGv\nTBnAsB7RvjeqhSYO7MjmnCP8bVkWie1aMqrXRYEOydRjdeenlDHV4L30PYx/aTVNwoJ59+4hdTZR\nAIgIfxjTk0s6RPDf89LJyrUGb+MeSxamQSguVp5atI1756bTt0ME7909lK7RzQMd1gULDw3mxYn9\naRwWwtTZ6ziaXxDokEw9ZcnC1HsnzxRy1+tf8ezyLMYP6MCc2wbSqmntb8j210Utw/nHzf349vuT\n/GKuNXgbd1iyMPXaviOehuyPM77jd1cn8uj1vQkLqX9/9imdW/P7axJZujWXZ5buCHQ4ph5q8A3c\n+QVFvJe+J9Bh1CotwkMZHh9NeGhwoEO5IOm7D3PH7DROnSni5ckDGB5fd9sn/DFpUCc25RzhmaU7\n6NmuBT/qaQ3epvo0+GRx4nQhv3l7U6DDqHVaNQnlJwM6MHFgpzo5LPaCDXuZ/u8NRLdoxOu3D6R7\nTN1vn/BFRPjjtb3Ytv8Yv5y3gfl3N6NrdLNAh2XqiQbfg7uoWNl/NN+FiOquXXkneG11Nosz91Os\nyoge0Uwa3InLu0URVMtHOy0uVv66dAd/W7qDlLjWvDCpP63rUfuEP/YePsXoZz+nReNQ3rt7KM3D\nrYe3KZ+/PbgbfLIw5dt35BT/WvMtb3y5mwPHT9OpTRMmDuzEjcmxtbKn86kzRfzq3xv4cNM+buwf\nyyPX1c/2CX+s2XWQm2euYViPaF6a1L/WJ3kTOJYsTLU5U1jMfzbvY86qbNKyD9EoJIgxSe24ZXAc\nvdq3DHR4AHx3JJ87Zqexee8RfvvjBG6/rDMS6BmLAuzVlV/z0PsZ/GJkd+4d2S3Q4Zhayt9k0eDb\nLIxvYSFBjElqz5ik9mTsPcqc1dnMX7+HeWk59O0YwS2DO3Fl77Y0CglMg/jGnMPcPiuNE6cLmXlL\nMqkJMQGJo7aZPCSOTXuO8pcl2+nZrgUjE+28mKqzKwtTJUdOFfD2uhxeW53NrgMnaN00jHEDOnDz\nwI7Etqq5BvEPNu7lv+dtIKp5I2ZOTib+ohY1duy6IL+giBtfWMU3B04wf9pQLo6yBm9zLquGMjWi\nuFhZufMAs1dlszRzPwAj4mO4ZXAnLu0a6VpduaryzNId/HXJDpI7teKFSf2JbNbIlWPVdXsOn2L0\n3z8nokko863B25RiycLUuD2HT/H66mzeXLubgyfO0DmyKRMHdWJsv1haNqm+L6j8Ak9D9gcb93FD\nv1j+dH2vgFWB1RWrdh5k4strSI2P5oWJ1uBt/o8lCxMwpwuL+M+m75i96hu++vYw4aFBXJvUnkmD\nO9Gz3YU1iO8/ms/U2Wls3HOE34yK56eXd2nwDdn+euXzr5nxQQb//cPu/CzVGryNhzVwm4BpFBLM\ntX3bc23f9mzec4Q5q7KZn76HuWt3079TK24Z3IlRvS6q9NXA5j1HuH1WGkfzC3hpUjI/tAbbSrl1\naByb9hzhz0u207N9C0bE2/kz/rMrC1Mjjpws4N/rdjNndTbZB08S2exsg3gn2kU09rn9wk37+OW8\ndNo09TRkJ7S1huyqyC8o4obnv+Dbgyd5b9pQuliDd4NXK6qhRGQU8AwQDMxU1cfKKPMT4CFAgQ2q\nepPXay2ATOBdVZ1W0bEsWdQNxcXKih15zFmVzbJtuQgwMiGGWwbHMbRrm/OqlFSVZ5dl8fTi7fTr\nGMGLk5KJam4N2Rci59BJRj+7kjZNw3j37qE0a2QVDA1ZwJOFiAQD24EfAjnAWmCCqmZ4lekGzANG\nqOohEYlW1Vyv158BooDvLVnUP7u/P8lra7KZt3Y3h04W0CWqKZMGdeKG/rG0CA8lv6CIX7+1kQUb\n9nJd3/Y8en3vOj+4YW3xxc4DTHr5S36YEMPzE/tZu08DVhuSxWDgIVW9wlm+H0BVH/Uq8wSwXVVn\nlrF9f2A68BGQbMmi/sovKOLDjfuYvTqbDbsP0yTM0+aRsfco6bsPM/2KHtw17GL7QqtmMz/bxcMf\nZjL9ih7cPbxroMMxAVIbGrjbA7u9lnOAgaXKdAcQkZV4qqoeUtWPRCQIeBqYBKSWdwARmQpMBejY\nsWP1RW5qVHhoMDf0j+WG/rFszDnM7FXZvL0uhyARXpjY3+aWdsltl3Zm854jPPXxNhLbtqj3Q7ib\nC+NmsijrZ2Dpy5gQoBswDIgFPhORXsBEYKGq7q7o16SqvgS8BJ4ri2qI2QRYn9gInroxggeuSiC/\noJiLWoYHOqR6S0R49Po+bN9/nHvmruf9aZcSF9k00GGZWsrNITlzgA5ey7HA3jLKvKeqBar6NbAN\nT/IYDEwTkW+Ap4BbROS8xnFTf0U0CbNEUQMahwXz4qT+hAQJU+d4xtcypixuJou1QDcR6SwiYcB4\nYEGpMvOB4QAiEomnWmqXqt6sqh1VNQ74FTBbVe9zMVZjGqwOrZvw7E39yMo9zvS3NlBfbqc31cu1\nZKGqhcA0YBGe21/nqeoWEZkhIqOdYouAgyKSASwHpqvqQbdiMsaUbWjXSO7/cQILN33H85/uDHQ4\nphayTnnGGMDTp+Xeuem8v3Ev/ztlAMN6WIN3Q+Dv3VANcxoxY8x5RITHb+hD/EUtuOeN9WQfPBHo\nkEwtYsnCGFOicVhwyTSsU2evswZvU8KShTHmHB1aN+HvE/qyI/cYv357ozV4G8CShTGmDJd1i+I3\no+L5cOM+XlyxK9DhmFrAkoUxpkxTL+/C1X3a8sRHW1mxPS/Q4ZgAs2RhjCmTiPDE2D50j2nOz95Y\nz7cHTwY6JBNAliyMMeVqEhbCi5P6AzB1Thonz1iDd0NlycIYU6FObZrytwl92b7/GL9+yxq8GypL\nFsYYn37QPYrpV8TzwcZ9jPrrZ8xZnc1xu622QbFkYYzxy50/6MLTN15CaIjwu/mbGfSnpTz43may\nco8FOjRTA2y4D2NMpagq6bsPM2dVNh9s3MeZomKGXNyGWwZ3YmRCDCHB9hu0Lgn4THk1zZKFMTXv\n4PHTvJm2m9dXf8uew6e4qEU4Nw3syPiUDkQ3tyHm6wJLFsaYGlNUrCzbmsvsVd/w2Y4DhAYLo3q1\n5ZbBnUju1MqmxK3FasO0qsaYBiI4SPhhYgw/TIxhV95xXlv9Lf9et5v3N+wl/qLm3DI4jjFJ7Wja\nyL5y6iq7sjDGuOLkmULeS9/L7FXZZO47SvNGIdzQP5ZJgztxcVSzQIdnHFYNZYypFVSVr749xOxV\n2SzctI+CIuXSrpFMGtyJ1PhoaxAPMEsWxphaJ+/Yad5c+y2vr/mWfUfyadcynJsHdWLcgA5ENmsU\n6PAaJEsWxphaq7ComCWZucxZ/Q0rsw4SFhzElb0vYtLgOPp1jLAG8RpkDdzGmForJDiIUb0uYlSv\ni8jKPc5rq7N5e10O89P30rNdCyYN6sSYpPY0DgsOdKjG4eqVhYiMAp4BgoGZqvpYGWV+AjwEKLBB\nVW8SkSTgeaAFUAQ8oqpvVnQsu7Iwpm47cbqQd9fvYc6qbLbtP0aL8BBuTO7ApEGdiItsGujw6q2A\nV0OJSDCwHfghkAOsBSaoaoZXmW7APGCEqh4SkWhVzRWR7oCq6g4RaQesAxJU9XB5x7NkYUz9oKqs\n/eYQs1d9w0ebv6OwWLm8exS3DOrE8PhogoOsiqo61YZqqBQgS1V3OQHNBcYAGV5l7gCeU9VDAKqa\n6/y7/WwBVd0rIrlAFFBusjDG1A8iQkrn1qR0bk3u0Xze+HI3//oym9tnpxHbqjE3D/Q0iLduGhbo\nUBsUN+9Zaw/s9lrOcdZ56w50F5GVIrLaqbY6h4ikAGHATtciNcbUStEtwrl3ZDc+/80I/nFzP2Jb\nNebxj7bygyeXs/9ofqDDa1DcTBZlXSuWrvMKAboBw4AJwEwRiSjZgUhbYA5wq6oWn3cAkakikiYi\naXl5Nu2jMfVVaHAQV/Zuy9ypg5n308Ecyy/k44z9gQ6rQXEzWeQAHbyWY4G9ZZR5T1ULVPVrYBue\n5IGItAA+BB5Q1dVlHUBVX1LVZFVNjoqKqvY3YIypfQbEtaJTmyYszbRkUZPcTBZrgW4i0llEwoDx\nwIJSZeYDwwFEJBJPtdQup/y7wGxV/beLMRpj6hgRITU+hi92HrRpXmuQa8lCVQuBacAiIBOYp6pb\nRGSGiIx2ii0CDopIBrAcmK6qB4GfAJcDU0Qk3XkkuRWrMaZuSU2I5kxhMZ/vOBDoUBoM68FtjKlz\nzhQW0/+Pi7mqT1seu6FPoMOp0/y9ddZG8DLG1DlhIUFc3j2KpVtzKS6uHz94aztLFsaYOik1IZq8\nY6fZtOdIoENpECxZGGPqpGE9ogkSWLo1N9ChNAiWLIwxdVLrpmH069jKbqGtIZYsjDF1VmpCDFv2\nHmXfkVOBDqXes2RhjKmzRiZEA7DMqqJcZ8nCGFNndY1uRofWjVmaacnCbZYsjDF11tne3CuzDnDq\nTFGgw6nXLFkYY+q0kQkxnC4sZmWW9eZ2kyULY0ydltK5Nc0ahbB0q90V5SZLFsaYOs3TmzuSpZnW\nm9tNliyMMXVeanwMucdOs2Xv0UCHUm9ZsjDG1HnDekQhAkusg55rLFkYY+q8Ns0aeXpzW7uFayxZ\nGGPqhdSEaDbvOcp3R2xubjdYsjDG1Aup8TGA9eZ2iyULY0y90D2mGbGtGrPMqqJc4VeyEJG3ReQq\nEbHkYoyplUSEkQkxfJ51gPwC681d3fz98n8euAnYISKPiUi8izEZY0yVjIiPJr/AenO7wa9koapL\nVPVmoB/wDbBYRL4QkVtFJLS87URklIhsE5EsEbmvnDI/EZEMEdkiIv/yWj9ZRHY4j8mVe1vGmIZo\nYJfWNA0LtgmRXOB3tZKItAGmALcD64Fn8CSPxeWUDwaeA34MJAITRCSxVJluwP3AUFXtCfzcWd8a\neBAYCKQAD4pIq8q8MWNMw9MoJJjLu0exLDMXVevNXZ38bbN4B/gMaAJco6qjVfVNVf0Z0KyczVKA\nLFXdpapngLnAmFJl7gCeU9VDAKp69ufAFcBiVf3eeW0xMKoyb8wY0zCNiI/mu6P51pu7mvl7ZfGs\nqiaq6qOqus/7BVVNLmeb9sBur+UcZ5237kB3EVkpIqtFZFQltjXGmPMMj49GBJvjopr5mywSRCTi\n7IKItBKRu3xsI2WsK31dGAJ0A4YBE4CZznH82RYRmSoiaSKSlpeX5yMcY0xDENmsEUkdIqw3dzXz\nN1ncoaqHzy44VUN3+NgmB+jgtRwL7C2jzHuqWqCqXwPb8CQPf7ZFVV9S1WRVTY6KivLzrRhj6ruR\nCTFszDnC/qPWm7u6+JssgkSk5Ne+03gd5mObtUA3EeksImHAeGBBqTLzgeHOPiPxVEvtAhYBP3Ku\nYFoBP3LWGWOMT6nO3NzL7a6oauNvslgEzBORVBEZAbwBfFTRBqpaCExzts0E5qnqFhGZISKjvfZ7\nUEQygOXAdFU9qKrfA3/Ek3DWAjOcdcYY41OPmOa0j2jMEmu3qDbiz+1lTs/tnwKpeNoTPgZmqmqt\n6SaZnJysaWlpgQ7DGFNL/P69zcxL2036739EeGhwoMOptURkXQU3KpXwt1Nesao+r6pjVfUGVX2x\nNiUKY4wpLTUhhvyCYlbtPBjoUOoFf/tZdBORt5ye1rvOPtwOzhhjqmpg59Y0CQu2CZGqib9tFv+L\nZ3yoQjwN0rOBOW4FZYwxFyo8NJjLukWybKv15q4O/iaLxqq6FE8bR7aqPgSMcC8sY4y5cKkJMew7\nkk/GPuvNfaH8TRb5TiP3DhGZJiLXAdEuxmWMMRdseA/rzV1d/E0WP8czLtQ9QH9gImAjwRpjarWo\n5o24JDbCRqGtBj6ThdMB7yeqelxVc1T1VueOqNU1EJ8xxlyQkQnRbNh9mNxj1pv7QvhMFs4tsv29\ne3AbY0xdMcKZm9t6c18Yf6uh1gPvicgkEbn+7MPNwIwxpjoktG1Ou5bh1m5xgUL8LNcaOMi5d0Ap\n8E61R2SMMdVIREhNiOGtdTnkFxRZb+4q8itZqOqtbgdijDFuGZEQzZzV2azadZDhPexGzqrwK1mI\nyP9SxnwSqvr/qj0iY4ypZoO7tKFJWDDLMnMtWVSRv20WHwAfOo+lQAvguFtBGWNMdQoPDebSrpEs\nzdxvvbmryN9qqLe9l0XkDWCJKxEZY4wLUhOi+ThjP5n7jpHYrkWgw6lz/L2yKK0b0LE6AzHGGDcN\nj/dUPy2z6VarxN9RZ4+JyNGzD+B94DfuhmaMMdUnunk4l3SIsAmRqsjfaqjmbgdijDFuS42P5i9L\ntpN37DRRzRsFOpw6xd8ri+tEpKXXcoSIXOteWMYYU/1SE6JRheXb7Oqisvxts3hQVY+cXVDVw8CD\n7oRkjDHuSGzbgrYtw1lqEyJVmr/Joqxy/vb+NsaYWkFEGBEfzWc7DnC60GaGrgx/k0WaiPxZRC4W\nkS4i8hdgna+NRGSUiGwTkSwRua+M16eISJ6IpDuP271ee0JEtohIpoj8zQYyNMZUh5EJMZw8U8Tq\nXd8HOpQ6xd9k8TPgDPAmMA84Bdxd0QbO0ObPAT8GEoEJIpJYRtE3VTXJecx0th0CDAX6AL2AAcAP\n/IzVGGPKNfjiNoSHBllVVCX5ezfUCeC8KwMfUoAsVd0FICJzgTFAhj+HBMKBMECAUMD+Z40xF8zT\nmzuKpZm5/GG0YpUW/vH3bqjFIhLhtdxKRBb52Kw9sNtrOcdZV9oNIrJRRN4SkQ4AqroKWA7scx6L\nVDWzjLimikiaiKTl5eX581aMMYaRCdHsOXyKbfuPBTqUOsPfaqhI5w4oAFT1EL7n4C4rXZcelOV9\nIE5V++AZPmQWgIh0BRKAWDwJZoSIXH7ezlRfUtVkVU2Oiory860YYxq6EU5vbpvjwn/+JotiESkZ\n3kNE4ihjFNpScoAOXsuxwF7vAqp6UFVPO4v/xDO/N8B1wGpnKtfjwH+AQX7GaowxFYpuEU6f2JbW\nblEJ/iaL/wE+F5E5IjIH+BS438c2a4FuItJZRMKA8cAC7wIi0tZrcTRwtqrpW+AHIhIiIqF4GrfP\nq4YyxpiqSo2PYf3uwxw4ftp3YeNfslDVj4BkYBueO6L+G88dURVtUwhMAxbh+aKfp6pbRGSGiIx2\nit3j3B67AbgHmOKsfwvYCWwCNgAbVPX9yrwxY4ypSElvbpub2y/iz9juTv+He/FUJaXjqRJapaoj\nKtywBiUnJ2taWlqgwzDG1BGqyuBHl9G3YwTPT+zve4N6SkTWqWqyr3L+VkPdi6evQ7aqDgf6Anb7\nkTGmzhIRRiREs2J7nvXm9oO/ySJfVfMBRKSRqm4FergXljHGuC81PpoTZ4pYY725ffI3WeQ4/Szm\nA4tF5D1K3dlkjDF1zdCukYSHBrHM2i188reB+zpVPayqDwG/A14GbIhyY0yddnZu7iU2N7dPlZ5W\nVVU/VdUFqnrGjYCMMaYmjYiPIefQKbbvPx7oUGq1qs7BbYwx9UJqgtOb2+bmrpAlC2NMgxbTIpze\n7Vva0B8+WLIwxjR4I+Kj+erbQ3x/wmrXy2PJwhjT4I1MiLHe3D5YsjDGNHi92rcgpkUja7eogCUL\nY0yDd3Zu7hXbD3CmsDjQ4dRKliyMMQbPKLTHTxfy5dfWm7ssliyMMQZPb+5GIUEssTkuymTJwhhj\ngMZhwQztGsnSrdabuyyWLIwxxpGaEM3u70+RlWu9uUuzZGGMMY6zc3MvsQ5657FkYYwxjrYtG9Oz\nXQuW2S2057FkYYwxXlITYliXfYhD1pv7HJYsjDHGS2p8NMUKy7dZVZQ3V5OFiIwSkW0ikiUi95Xx\n+hQRyRORdOdxu9drHUXkYxHJFJEMEYlzM1ZjjAHo3b4lUc0bsdSG/jhHiFs7FpFg4Dngh0AOsFZE\nFqhqRqmib6rqtDJ2MRt4RFUXi0gzwLpVGmNcFxQkpMZH8+HGfZwpLCYsxCpgwN0rixQgS1V3ORMl\nzQXG+LOhiCQCIaq6GEBVj6vqSfdCNcaY/zMiPppjpwtZ+4315j7LzWTRHtjttZzjrCvtBhHZKCJv\niUgHZ1134LCIvCMi60XkSedK5RwiMlVE0kQkLS8vr/rfgTGmQbq0WyRhIUE2x4UXN5OFlLGudLfI\n94E4Ve0DLAFmOetDgMuAXwEDgC7AlPN2pvqSqiaranJUVFR1xW2MaeCahIUw9OI21pvbi5vJIgfo\n4LUcC+z1LqCqB1X1tLP4T6C/17brnSqsQmA+0M/FWI0x5hwjEmLIPniSnXknAh1KreBmslgLdBOR\nziISBowHFngXEJG2XoujgUyvbVuJyNnLhRFA6YZxY4xxTarTm3upDSwIuJgsnCuCacAiPElgnqpu\nEZEZIjLaKXaPiGwRkQ3APThVTapahKcKaqmIbMJTpfVPt2I1xpjS2kU0JrFtC2u3cLh26yyAqi4E\nFpZa93uv5/cD95ez7WKgj5vxGWNMRVITonlueRaHT54hoklYoMMJKLuB2BhjypGaEEOxwifb7G5L\nSxbGGFOOPu1bEtmskU2IhCULY4wpV1CQMCI+ik+351FQ1LAHkbBkYYwxFUhNiOFYvvXmtmRhjDEV\nuLSr9eYGSxbGGFOhpo1CGNylDcsa+Ci0liyMMcaHkQnRfH3gBDvzGu7c3JYsjDHGh+HWm9uShTHG\n+BLbqgnxFzVv0O0WliyMMcYPIxNiSMs+xJGTBYEOJSAsWRhjjB9GJERTVKx8sr1hXl1YsjDGGD8k\nxUYQ2SyswVZFWbIwxhg/BAUJw3tE88m23AbZm9uShTHG+Ck1IZqj+YWsyz4U6FBqnCULY4zx02Xd\noggLDmqQt9BasjDGGD81bRTCoIvbNMh2C0sWxhhTCanx0ew6cIJdDaw3tyULY4yphNQET2/uhjZW\nlCULY4yphLO9uRvahEiuJgsRGSUi20QkS0TuK+P1KSKSJyLpzuP2Uq+3EJE9IvKsm3EaY0xljIiP\nZu03hzhyquH05nYtWYhIMPAc8GMgEZggIollFH1TVZOcx8xSr/0R+NStGI0xpipSE2IoKlY+3d5w\n5uZ288oiBchS1V2qegaYC4zxd2MR6Q/EAB+7FJ8xxlRJUocI2jQNa1C30LqZLNoDu72Wc5x1pd0g\nIhtF5C0R6QAgIkHA08B0F+MzxpgqCQ4ShsdHszhjP6t2Hgx0ODXCzWQhZazTUsvvA3Gq2gdYAsxy\n1t8FLFTV3VRARKaKSJqIpOXlNZzLQWNM4P18ZDfaRTRm0strmPvlt4EOx3VuJoscoIPXciyw17uA\nqh5U1dPO4j+B/s7zwcA0EfkGeAq4RUQeK30AVX1JVZNVNTkqKqq64zfGmHLFtmrCO3cNYUjXSO57\nZxMz3s+gqLj07+H6w81ksRboJiKdRSQMGA8s8C4gIm29FkcDmQCqerOqdlTVOOBXwGxVPe9uKmOM\nCaQW4aG8MjmZW4fG8crKr7lt1lqO5dfPO6RcSxaqWghMAxbhSQLzVHWLiMwQkdFOsXtEZIuIbADu\nAaa4FY8xxrghJDiIB6/pySPX9eLzHQe4/h9f8O3Bk4EOq9qJav24bEpOTta0tLRAh2GMacC+yDrA\nf73+FUECL0zsz8AubQIdkk/eugPBAAASqUlEQVQisk5Vk32Vsx7cxhhTTYZ0jWT+3UNp1TSMiS+v\nYd7aCu/RqVMsWRhjTDXqHNmUd+8ayqAubfj12xt55MP60fBtycIYY6pZy8ah/O+UAUwe3Il/fvY1\nd8xOq/MN35YsjDHGBSHBQfxhTC/+eG0vPt2ex9jnV7H7+7rb8G3JwhhjXDRpUCdm3ZrCviOnGPPc\nStZ+832gQ6oSSxbGGOOyS7t5Gr5bNg7lpn+u5t9pda/h25KFMcbUgC5RzZh/11BSOrdm+lsbeXRh\nZp1q+LZkYYwxNaRlk1BevTWFiYM68uKKXfx0ThrHTxcGOiy/WLIwxpgaFBocxMPX9uYPo3uybGsu\nY5//gpxDtb/h25KFMcYEwOQhcbx6awp7Dp/i2udWsi67djd81+vhPgoKCsjJySE/Pz9AUZn6KDw8\nnNjYWEJDQwMdiqkHsnKPc/ustew9nM9jN/Tm+n6xNXp8f4f7CKmJYAIlJyeH5s2bExcXh0hZ02sY\nUzmqysGDB8nJyaFz586BDsfUA12jmzH/7qH812tf8ct5G9iRe5zpP+pBUFDt+s6q19VQ+fn5tGnT\nxhKFqTYiQps2bexq1VSriCZhzL4thQkpHXn+k5389LV1nKhlDd/1OlkAlihMtbO/KeOG0OAg/nRd\nLx68JpGlmfsZ+8Iq9hw+FeiwStT7ZFFbDBw4kKSkJDp27EhUVBRJSUkkJSXxzTffVGo/77zzDlu3\nbq308S+99FLS09Mrvd1ZTz31FP/617+qvH1NuPHGG9m1a1eZr505c4aJEyfSu3dvEhISeOKJJ8os\nt3PnTlJSUujatSs33XQTBQV1ezwfU7eICLcO7cwrUwaQ8/1Jxjy7knXZhwIdFmDJosasWbOG9PR0\nZsyYwbhx40hPTyc9PZ24uLhK7aeqyeJCFBQUMGfOHMaNG1ejx62sO++8kyeffLLM1+bOnQvApk2b\nSEtL4+9//zs5OTnnlZs+fTq//vWvycrKokmTJrz66qtuhmxMmYb1iOadu4bQJCyYCf9czfz1ewId\nkiWL2uA///kPgwcPpl+/fowbN44TJ04Ani+uxMRE+vTpw29+8xs+++wzFi5cyC9+8YsqXZWc9dpr\nr9G7d2969erFb3/725L1L774It27d2fYsGHcfvvt/PznPwdg8eLFDBgwgODgYABWr15Nnz59GDJk\nCNOnTycpKQnw/Cq/7LLL6Nu3L/3792fNmjUALFmyhOHDhzN27Fi6devGAw88wOzZsxkwYAB9+vQp\neR8TJ07k7rvvZvjw4Vx88cWsWLGCyZMnEx8fz2233VYS59SpU0lOTqZnz57MmDGjZP2wYcP46KOP\nKCoqOu89iwgnTpygqKiIU6dOER4eTvPmzc8pU1RUxIoVK7juuusAmDx5MvPnz6/SOTbmQnWLac78\nu4fSt0MEP38znScXbaU4gD2+6/XdUN7+8P4WMvYerdZ9JrZrwYPX9LygfeTm5vLYY4+xdOlSmjRp\nwiOPPMIzzzzDbbfdxsKFC9myZQsiwuHDh4mIiODKK69k7NixXHvttVU6Xk5ODg888ABpaWm0bNmS\nkSNH8sEHH3DJJZfw2GOP8dVXX9G0aVOGDRtGSkoKACtXrqR///4l+7j11luZNWsWKSkp/OpXvypZ\n37ZtWxYvXkx4eDhbt25l8uTJJQljw4YNZGZm0rJlS+Li4rjrrrtYu3YtTz/9NM8++yxPPfUUAEeO\nHGH58uW8/fbbXHPNNaxatYr4+Hj69evH5s2b6dWrF4899hitW7emsLCwJAklJiYSHBxMXFwcmzdv\n5pJLLjnnfY8fP54FCxbQtm1bTpw4wd/+9jdatmx5Tpm8vDwiIyNLkmJsbCx79gT+F51puFo3DWPO\nbQP5/XubeW75TrJyj/OXcUk0Cav5r267sgiwL774goyMDIYMGUJSUhKvv/4633zzDa1btyYoKIg7\n7riDd999l6ZNm1bL8dasWcOIESOIjIwkNDSUm266iRUrVpSsb9WqFWFhYYwdO7Zkm3379hEVFQXA\ngQMHOHPmTEkiuemmm0rKnT59mttuu41evXoxfvx4MjIySl4bOHAgMTExhIeH06VLF6644goAevfu\nfc4V0jXXXFOyvl27diQmJhIUFERiYmJJuTfeeIN+/frRr18/MjMzzzlOdHQ0e/fuPe99r1q1ivDw\ncPbs2cOuXbt4/PHHyc7OPqdMWX2OrDHbBFpYSBCPXt+bB65KYHHGfsY+v4q9AWj4djU9icgo4Bkg\nGJipqo+Ven0K8CRw9ufbs6o6U0SSgOeBFkAR8IiqvnkhsVzoFYBbVJVRo0YxZ86c815LS0tj8eLF\nzJ07l+eff56PP/643P14f4Fff/31/P73vy/3eJVZD9C4ceOSW0UrKvf000/ToUMHXnvtNQoKCmjW\nrFnJa40aNSp5HhQUVLIcFBREYWHheeW8y3iX27FjB8888wxffvklERERTJw48ZzbWPPz82ncuDFv\nvfUWDz/8MACvvvoqr7/+OldeeSWhoaHExMQwaNAg1q1bR6dOnUq2jY6O5sCBAxQVFREcHExOTg7t\n2rUr9/0aU1NEhNsv68LFUc342RvrGfPcSl6a1J++HVvVWAyuXVmISDDwHPBjIBGYICKJZRR9U1WT\nnMdMZ91J4BZV7QmMAv4qIhFuxRpIQ4YM4dNPPy25i+fEiRPs2LGDY8eOcfToUa6++mr+8pe/sH79\negCaN2/OsWPHzttPWFhYSaN5eYkCYNCgQSxfvpyDBw9SWFjI3Llz+cEPfsDAgQNZvnw5hw8fpqCg\ngHfeeadkm4SEBLKysgCIiooiNDSUs73lzzYcg6cKqW3btogIs2bNqjCxVNXRo0dp3rw5LVq0YN++\nfSxatOic13fs2EHPnj0ZO3Zsyfk4exfasmXLADh+/Dhr1qyhR48e52wbHBzMZZddxrvvvgvArFmz\nGDNmTLW/B2Oqani8p+E7PDSIcS+t5r30mqsmdbMaKgXIUtVdqnoGmAv49clT1e2qusN5vhfIBaJc\nizSAYmJiePnllxk3bhyXXHIJQ4YMYfv27Rw5coSrrrqKSy65hBEjRvDnP/8ZgAkTJvCnP/2pyg3c\nsbGxzJgxg2HDhpGUlMSgQYO46qqr6NixI9OnTyclJYUf/ehH9OzZs6RO/8orr+TTTz8t2ccrr7zC\nrbfeypAhQwgKCiopN23aNGbOnMmgQYPIzs4+58qguvTr14/ExER69erFHXfcwdChQ0te27t3Ly1b\ntiypMvN2zz338P3339OzZ09SUlK488476dnTc7V5xRVXkJubC8CTTz7J448/TteuXTl+/DhTpkyp\n9vdgzIXoHtOc+XcNJSk2gnvnpvPnj7fVTMO3qrryAMbiqXo6uzwJTzWTd5kpwD5gI/AW0KGM/aQA\nmUBQRcfr37+/lpaRkXHeOlO+Y8eOqarqmTNn9Mc//rEuWLCg5LVrrrlGd+7ceU45VdWHH35Yf/nL\nX9ZsoOV44okn9NVXX62RY9nflgm0/IJC/dW8dO30mw/0v15L08Ki4irtB0hTP77T3byyKKtlsHT6\nex+IU9U+wBJg1jk7EGkLzAFuVdXi8w4gMlVE0kQkLS8vr5rCbrh+97vf0bdvX/r06UOPHj24+uqr\nS157/PHHSxqOFyxYQFJSEr169WLVqlXcf//9gQr5HG3atGHixImBDsOYGtEoJJgnxvbht1fGc3FU\nM4JdHkvKtVFnRWQw8JCqXuEs3w+gqo+WUz4Y+F5VWzrLLYBPgEdV9d++jlfWqLOZmZkkJCRcyNsw\npkz2t2XqC39HnXXzymIt0E1EOotIGDAeWOBdwLlyOGs0nuomnPLvArP9SRTGGGPc5dqts6paKCLT\ngEV4bp19RVW3iMgMPHVkC4B7RGQ0UAh8j6cNA+AnwOVAG+f2WoApqlrpwY1U1e6VN9XKratxY2qz\nej350ddff03z5s1tmHJTbdSZz+LYsWM2n4WpF2zyIzy3iebk5GCN36Y6nZ0pz5iGpF4ni9DQUPv1\nZ4wx1cDGhjLGGOOTJQtjjDE+WbIwxhjjU725G0pE8oBsnwXLFwkcqKZwqpPFVTkWV+VYXJVTH+Pq\npKo+x96rN8niQolImj+3j9U0i6tyLK7KsbgqpyHHZdVQxhhjfLJkYYwxxidLFv/npUAHUA6Lq3Is\nrsqxuCqnwcZlbRbGGGN8sisLY4wxPjXYZCEiN4rIFhEpFpFy7yIQkVEisk1EskTkvhqIq7WILBaR\nHc6/Zc7ILiJFIpLuPBaUVaaa4qnw/YtIIxF503l9jYjEuRVLJWKaIiJ5Xufndrdjco77iojkisjm\ncl4XEfmbE/dGEelXS+IaJiJHvM5X+ZO4V29cHURkuYhkOp/Fe8soU+PnzM+4avyciUi4iHwpIhuc\nuP5QRhn3Po/+TKdXHx9AAtADzwRLyeWUCQZ2Al2AMGADkOhyXE8A9znP7wMeL6fc8Ro4Rz7fP3AX\n8ILzfDzwZi2IaQqlpvCtob+py4F+wOZyXr8S+A+eWSQHAWtqSVzDgA8CcL7aAv2c582B7WX8X9b4\nOfMzrho/Z845aOY8DwXWAINKlXHt89hgryxUNVNVt/kolgJkqeouVT0DzAXGuBzaGP5vetlZwLUu\nH68i/rx/73jfAlLF3fHgA/F/4hdVXYFnXpbyjMEzoZeq6mogotQEYIGKKyBUdZ+qfuU8P4Zn8rP2\npYrV+DnzM64a55yD485iqPMo3ejs2uexwSYLP7UHdnst5+D+H02Mqu4Dzx8tEF1OuXBn/vHVIuJW\nQvHn/ZeUUdVC4AjQxqV4/I0J4Aan2uItEengYjyVEYi/J38Ndqo3/iMiPWv64E51SV88v5a9BfSc\nVRAXBOCciUiwiKQDucBiVS33fFX357FeD1EuIkuAi8p46X9U9T1/dlHGugu+fayiuCqxm46quldE\nugDLRGSTqu680NhK8ef9u3KOKuDP8d4H3lDV0yJyJ55fWiNcjMlfNX2u/PUVniEfjovIlcB8oFtN\nHVxEmgFvAz9X1aOlXy5jkxo5Zz7iCsg5U9UiIElEIoB3RaSXqnq3Rbl2vup1slDVkRe4ixzA+1dp\nLLD3AvdZYVwisl9E2qrqPudyO7ecfex1/t0lIp/g+fVT3cnCn/d/tkyOiIQALXG3ysNnTKp60Gvx\nn8DjLsZTGa78PV0o7y9CVV0oIv8QkUhVdX0MJBEJxfOF/LqqvlNGkYCcM19xBfKcOcc87HzuRwHe\nycK1z6NVQ1VsLdBNRDqLSBieBiPX7jxyLAAmO88nA+ddAYlIKxFp5DyPBIYCGS7E4s/79453LLBM\nndY1l/iMqVSd9mg8dc61wQLgFucOn0HAkbNVjoEkIhedrdcWkRQ83wsHK96qWo4rwMtApqr+uZxi\nNX7O/IkrEOdMRKKcKwpEpDEwEthaqph7n8eabM2vTQ/gOjxZ+DSwH1jkrG8HLPQqdyWeuyF24qm+\ncjuuNsBSYIfzb2tnfTIw03k+BNiE506gTcBtLsZz3vsHZgCjnefhwL+BLOBLoEsNnCNfMT0KbHHO\nz3Igvob+pt4A9gEFzt/WbcCdwJ3O6wI858S9iXLuwgtAXNO8ztdqYEgNxXUpniqSjUC687gy0OfM\nz7hq/JwBfYD1Tlybgd8762vk82g9uI0xxvhk1VDGGGN8smRhjDHGJ0sWxhhjfLJkYYwxxidLFsYY\nY3yyZGFMJYjIcd+lKtz+LafXPSLSTEReFJGdziiiK0RkoIiEOc/rdadZU7dYsjCmhjjjBwWr6i5n\n1Uw8vWu7qWpPPKPlRqpngMSlwLiABGpMGSxZGFMFTo/iJ0Vks4hsEpFxzvogZ+iHLSLygYgsFJGx\nzmY34/TIF5GLgYHAA6paDJ6hW1T1Q6fsfKe8MbWCXeYaUzXXA0nAJUAksFZEVuAZeiUO6I1nxOBM\n4BVnm6F4elMD9ATS1TMwXFk2AwNcidyYKrArC2Oq5lI8I9sWqep+4FM8X+6XAv9W1WJV/Q7PcCNn\ntQXy/Nm5k0TOiEjzao7bmCqxZGFM1ZQ3oUxFE82cwjN2D3jGFbpERCr6DDYC8qsQmzHVzpKFMVWz\nAhjnTEYThWfq0i+Bz/FMvBQkIjF4pt88KxPoCqCeuUfSgD94jV7aTUTGOM/bAHmqWlBTb8iYiliy\nMKZq3sUz+ucGYBnwa6fa6W08I7tuBl7EM8PaEWebDzk3edyOZxKsLBHZhGfujbNzNQwHFrr7Fozx\nn406a0w1E5Fm6plBrQ2eq42hqvqdMwfBcme5vIbts/t4B7hffc8Tb0yNsLuhjKl+HziT1IQBf3Su\nOFDVUyLyIJ55kr8tb2NnUqf5lihMbWJXFsYYY3yyNgtjjDE+WbIwxhjjkyULY4wxPlmyMMYY45Ml\nC2OMMT5ZsjDGGOPT/wcd9fvq9EB4EAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x3061b550>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "accuracy_s1 =np.array(accuracy_s).reshape(len(C_s),len(gamma_s))\n",
    "x_axis = np.log10(C_s)\n",
    "for j, gamma in enumerate(gamma_s):\n",
    "    plt.plot(x_axis, np.array(accuracy_s1[:,j]), label = ' Test - log(gamma)' + str(np.log10(gamma)))\n",
    "\n",
    "plt.legend()\n",
    "plt.xlabel( 'log(C)' )                                                                                                      \n",
    "plt.ylabel( 'accuracy' )\n",
    "plt.savefig('RBF_SVM_Otto.png' )\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# 预测 ,使用accuracy: 0.65 C=0.774264 gamma=0.000000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 在训练集是那个利用SVC训练\n",
    "SVC3 =  SVC( C = 46.415888 , kernel='rbf', gamma = 'auto')\n",
    "SVC3 = SVC3.fit(X_train, y_train)\n",
    "\n",
    "# 预测\n",
    "test_data_pred(SVC3,test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
